# Use the file here to make your favorite movie lists through media

import media
import fresh_tomatoes

# In the three variables below they all call the class movie (in media).
# There are three arguments in the class which are title, image url, and
# youtube video url.
oblivion = media.movie('Oblivion',
                       'https://images-na.ssl-images-amazon.com/images/M/MV5'
                       'BMTQwMDY0MTA4MF5BMl5BanBnXkFtZTcwNzI3MDgxOQ@@._'
                       'V1_UY1200_CR64,0,630,1200_AL_.jpg',
                       'https://www.youtube.com/watch?v=XmIIgE7eSak')

pacific_rim = media.movie('Pacific Rim: Uprising',
                          'https://resizing.flixster.com/MbiJhGRyRcft'
                          'BfFpMg61V2OuEdQ=/300x300/v1.bTsxMTE3MzkzM'
                          'ztqOzE3NDU0OzEyMDA7ODAwOzEyMDA',
                          'https://www.youtube.com/watch?v=5guMumPFBag')

interstellar = media.movie('Interstellar',
                           'https://images-na.ssl-images-amazon.com/'
                           'images/M/MV5BMjIxNTU4MzY4MF5BMl5BanBnXkFtZTg'
                           'wMzM4ODI3MjE@._V1_UY1200_CR69,0,630,1200_AL_',
                           '.jpghttps://www.youtube.com/watch?v=0vxOhd4qlnA')

# The movie list was made into a string to be taken into the function
# fresh_tomatoes.open_movies_page()
movie_list = [oblivion, pacific_rim, interstellar]

fresh_tomatoes.open_movies_page(movie_list)


# <--imports-->
import time
import requests
import tkinter as tk

# define weather api
def getWeather(astro):
      city = textfield.get()
      api = "http://api.openweathermap.org/data/2.5/weather?q=" + city +"&appid=9eb2a3f199c1fce2d9638830c7d1c7ae"
      json_data = requests.get(api).json()
      condition = json_data['weather'][0]['main']
      temp = int(json_data['main']['temp'] - 273.15)
      min_temp = int(json_data['main']['temp_min'] - 273.15)
      max_temp = int(json_data['main']['temp_max'] - 273.15)
      pressure = json_data['main']['pressure']
      hum = json_data['main']['humidty']
      wind = json_data['wind']['speed']

      final_info = condition + "\n" + str(temp) + "°C"
      final_data = "\n" + "Max Temp:" + str(max_temp) + "\n" + "Min Temp:" + str(min_temp) + "\n" + "Pressure:" + str(pressure) + "\n" + "Wind:" + str(wind) + "\n" "Humidity:" + str(hum)

      label1.config(text = final_info)
      label2.config(text = final_data)

# INTERFACE
astro = tk.Tk()
astro.geometry("300x200")
astro.title("ASTRO-WEATHER-APP")

f = ("poppins", 15, "bold")
t = ("poppins", 25, "bold")

textfield = tk.Entry(astro, font = t)
textfield.pack(pady = 20)
textfield.focus()
textfield.bind('<return>', getWeather)
label1 = tk.Label(astro, font = t)
label1.pack()
label2 = tk.Label(astro, font = f)
label2.pack()

astro.mainloop()


import socket

s = socket.socket()

port = 12345

s.connect(('127.0.0.1', port))

a = input('Enter the first number:')
b = input('Enter the second number:')

ip = str(a) + '+' + str(b)

ip = ip.encode('utf-8');

s.send(ip)

string = s.recv(1024)

string = string.decode('utf-8')

print(string)

s.close()


#Author: Karim Mukaddem and Batuhan Yildirim

from PIL import Image
from pytesseract import image_to_string, image_to_boxes
from skimage import exposure
from keras.models import load_model
import scipy.stats as stats
from operator import itemgetter
import os

from .contour_detections import *
from .img_utils import psnr

def read_image(thresholdedimg, unit):
    '''Read text from an image using pytesseract. Note: pytesseract reads PIL format images,
    so we convert our cv2 (numpy array) for this step.

    :param numpy.ndarray thresholdedimg: input image to be read.
    :param string unit: "nm" or "um" depending on what we're looking for.

    :return string greekengtextfromimage: text that was read.
     '''

    # convert input to appropriate format.
    imagetoread = Image.fromarray(thresholdedimg)

    # Only look for these characters.
    if unit == "um":
        cfg = 'pytess_config_um.txt'
    else:
        cfg = "pytess_config_nm.txt"

    # Read image using English alphabet.
    greekengtextfromimage = image_to_string(imagetoread, lang="eng", config=os.path.join(os.path.dirname(os.path.abspath(__file__)), cfg))

    text_boxes = image_to_boxes(imagetoread, lang="eng", config=os.path.join(os.path.dirname(os.path.abspath(__file__)), cfg))

    return greekengtextfromimage, text_boxes

def threshold_for_text_reading(img, imgmean, imgstdev, threshold_to, thresholdvalue=None):
    '''Thresholds (binarize/filter) an image to facilitate either white or black text reading.
    :param numpy.ndarray img: image to be read
    :param int imgmean: mean pixel intensity in image
    :param float imgstdev: standard deviation of pixel intensities in image
    :param int threshold_to: 0 if looking for white text, 1 if looking for black text.
    :param int thresholdvalue: Optionally provide an exact threshold value [0,255]

    :return numpy.ndarray thresholdedimg: Filtered image.
    '''

    # Looking for white text.
    if threshold_to == 0:

        if thresholdvalue == None:
            thresholdvalue = imgmean + (3.5 * imgstdev)

        thresholdmax = 235
        if thresholdvalue > thresholdmax:
            thresholdvalue = thresholdmax

        ret, thresholdedimg = cv2.threshold(img, thresholdvalue, 255, threshold_to)

    # Look for black text.
    elif threshold_to == 1:

        if thresholdvalue == None:
            thresholdvalue = imgmean - (4 * imgstdev)

        thresholdmin = 20
        if thresholdvalue < thresholdmin:
            thresholdvalue = thresholdmin

        ret, thresholdedimg = cv2.threshold(img, thresholdvalue, 255, threshold_to)

    return thresholdedimg

def detect_scale_bar_and_inlays(gimg, imgmean, imgstdev, rows, cols, show = False):
    '''Visually detects scale bar of an image to determine the physical distance a pixel represents.
    :param numpy.ndarray gimg: input image
    :param int imgmean: mean pixel intensity in image
    :param float imgstdev: standard deviation of pixel intensities in image
    :param int rows: number of rows in image
    :param int cols: number of cols in image
    :param bool show: to monitor activities step by step

    :return tuple scalebar: (x,y,w,h) top left corner of scalebar, width, height.
    :return float scalevalue: number associated with scale bar.
    :return float conversion: unit of scalevalue 10e-6 for um, 10e-9 for nm.
    :return list inlaycoords: list of tuples in same format of obstructive inlays in image.

    '''

    scalebar, scalevalue, conversion = None, None, 1

    # Create new copy of image to work with.
    scalebarimg = gimg.copy()

    # Frame around edges.
    frame_constant = 3
    cv2.rectangle(scalebarimg, (0, 0), (frame_constant, rows), (0, 0, 0), -1)
    cv2.rectangle(scalebarimg, (0, rows - frame_constant), (cols, rows), (0, 0, 0), -1)
    cv2.rectangle(scalebarimg, (0, 0), (cols, frame_constant), (0, 0, 0), -1)
    cv2.rectangle(scalebarimg, (cols - frame_constant, 0), (cols, rows), (0, 0, 0), -1)

    # Threshold image and look for white text.
    thresholdedimg = threshold_for_text_reading(scalebarimg, imgmean, imgstdev, 0)
    scalebar, scalevalue, conversion, inlaycoords = find_text_and_bar(thresholdedimg, gimg, rows, cols, show=show, printer=show)

    # If nothing is found repeat process looking for black text.
    if scalebar == None and scalevalue == None and conversion == 1:
        cv2.rectangle(scalebarimg, (0, 0), (frame_constant, rows), (255, 255, 255), -1)
        cv2.rectangle(scalebarimg, (0, rows - frame_constant), (cols, rows), (255, 255, 255), -1)
        cv2.rectangle(scalebarimg, (0, 0), (cols, frame_constant), (255, 255, 255), -1)
        cv2.rectangle(scalebarimg, (cols - frame_constant, 0), (cols, rows), (255, 255, 255), -1)

        thresholdedimg_black = threshold_for_text_reading(scalebarimg, imgmean, imgstdev, 1)
        scalebar, scalevalue, conversion, inlaycoords = find_text_and_bar(thresholdedimg_black, gimg, rows, cols, show=show, printer=show, black_text=True)

    return scalebar, scalevalue, conversion, inlaycoords

def srcnn_predict(image, model, log=1.2):
    image = np.expand_dims(np.expand_dims(image, 0), 3)
    sr_image = model.predict(image)[0, :, :, 0].astype(np.uint8)
    sr_image = exposure.adjust_log(sr_image, 1.2)
    return sr_image

def find_text_and_bar(thresholdedimg, gimg, rows, cols, show=False, printer=False, black_text=False):
    '''Locates scale information (bar, value and unit) in a thresholded image. (actual reading part of parent function).
    :param numpy.ndarray thresholdedimg: filtered input image, filtered for black or white text reading.
    :param numpy.ndarray gimg: original image in grayscale.
    :param int rows: number of rows in image.
    :param int cols: number of columns in image.
    :param bool show: display images step by step.
    :param bool printer: print what's going on step by step.
    :param bool black_text: True if we're looking for black text/bar, False if white.

    :return tuple scalebar: (x,y,w,h) top left corner of scalebar, width, height.
    :return float scalevalue: number associated with scale bar.
    :return float conversion: unit of scalevalue 10e-6 for um, 10e-9 for nm.
    :return list inlaycoords: list of tuples in same format of obstructive inlays in image.

    '''

    scalebar = None
    conversion = 1
    scalevalue = None
    boxes = None
    inlaycoords = []
    inlay = None
    interp_scale = 3

    model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'srcnn/srcnn_model.h5')

    srcnn = load_model(model_path, custom_objects={'psnr': psnr})

    # Make regions including text/info into large blobs so they can be easily identified.
    blurred_thresholdedimg = cv2.medianBlur(thresholdedimg, 1)
    dilationkernel = np.ones((6, 8), np.uint8)
    dilatedimg = cv2.dilate(blurred_thresholdedimg, dilationkernel, iterations=3)

    if black_text == False:
        restrict_rectangles_color = "white"
    else:
        restrict_rectangles_color = "black"

    filteredvertices, boundingrectanglewidths, boundingrectanglecoords = find_draw_contours \
        (dilatedimg, 151, minarea=10, testing=show, annotate=False, bounding_rect=True, restrict_rectangles=False)

    potential_scale_bar_regions = []
    area_index = 1

    for i in boundingrectanglecoords:

        if printer == True:
            print(area_index, " out of ", len(boundingrectanglecoords))

        is_box_in_box = False
        region_size = int(i[2]) * int(i[3])

        if region_size > rows * cols / 100:

            # First checking if the region we've identified is a box containing information. As opposed to text put directly on the image.

            box_filteredvertices, box_boundingrectanglewidths, box_boundingrectanglecoords = find_draw_contours \
                (thresholdedimg[i[1]:i[1] + i[3], i[0]:i[0] + i[2]], 151, minarea=10, testing=show, annotate=show, bounding_rect=True, restrict_rectangles=True)

            box_boundingrectanglecoords.sort(key=itemgetter(2))
            box_boundingrectanglecoords.reverse()

            for potential_box in box_boundingrectanglecoords:

                box_in_box = gimg[i[1] + potential_box[1]:i[1] + potential_box[1] + potential_box[3], i[0] + potential_box[0]:i[0] + potential_box[0] + potential_box[2]]

                box_in_box_filteredvertices = find_draw_contours \
                    (box_in_box, 151, minarea=10, testing = show, annotate = show)

                # Check that we've found box and not just the scale bar by mistake. By making sure shapes exist within this region.
                if len(box_in_box_filteredvertices) > 2:
                    i = (i[0] + potential_box[0], i[1] + potential_box[1], potential_box[2], potential_box[3])

                    is_box_in_box = True

                # Only have to check widest box, so we can break after a single a loop.
                break

            if printer == True:
                print("is_box_in_box", is_box_in_box)

            # These get set to None each loop unless both can be identified simultaneously further down.
            conversion = 1
            boxes = None
            scalevalue = None

            if is_box_in_box == True:
                black_text = True

            # Cannot determine what exact binarization threshold works for each region, so we scan through 35,0 as thresholds in increments of 5 until
            # something matching our text criteria is found.

            if black_text == False:

                inverted = (255 - gimg[i[1]:i[1] + i[3], i[0]:i[0] + i[2]])
            else:
                inverted = gimg[i[1]:i[1] + i[3], i[0]:i[0] + i[2]]

            # show_image(inverted)
            h, w = inverted.shape[:2]
            inverted_cubic = cv2.resize(inverted, (w * interp_scale, h * interp_scale), interpolation=cv2.INTER_CUBIC)
            inverted_srcnn = srcnn_predict(inverted, srcnn)
            opening_kernel = np.ones((5, 3), np.uint8)
            inverted_srcnn = cv2.morphologyEx(inverted_srcnn, cv2.MORPH_OPEN, opening_kernel)

            for inverted in [inverted_cubic, inverted_srcnn]:
                if not conversion or not scalevalue:
                    for pixel_value in reversed(range(5, 40, 5)):

                        if not conversion or not scalevalue:

                            uv, region_surrounding_text_reading = cv2.threshold(inverted, pixel_value, 255, 0)
                            region_surrounding_text_reading = cv2.medianBlur(region_surrounding_text_reading, 1)

                            greekengtextfromimage_edited_nm, edited_nm_boxes = read_image(region_surrounding_text_reading, unit="nm")
                            greekengtextfromimage_edited_um, edited_um_boxes = read_image(region_surrounding_text_reading, unit="um")
                            if printer == True:
                                print("greek&eng_edited_nm")
                                print(greekengtextfromimage_edited_nm)

                                print("greek&eng_edited_um")
                                print(greekengtextfromimage_edited_um)

                            if show == True:
                                show_image(region_surrounding_text_reading, 0, "blurred")

                            # Check raw images only once, at the beginning.
                            if pixel_value == 35:
                                if show == True:
                                    show_image(inverted, 0, 'raw')

                                greekengtextfromimage_raw_nm, raw_nm_boxes = read_image(inverted, unit="nm")
                                greekengtextfromimage_raw_um, raw_um_boxes = read_image(inverted, unit="um")

                                if printer == True:
                                    print("greek&eng_raw_nm")
                                    print(greekengtextfromimage_raw_nm)

                                    print("greek&eng_raw_um")
                                    print(greekengtextfromimage_raw_um)

                                if show == True:
                                    show_image((255 - inverted), 0, '255-inverted')

                                greekengtextfromimage_raw_inverted_nm, inverted_nm_boxes = read_image((255 - inverted), unit="nm")
                                greekengtextfromimage_raw_inverted_um, inverted_um_boxes = read_image((255 - inverted), unit="um")

                                if printer == True:
                                    print("greek&eng_raw_inverted_nm")
                                    print(greekengtextfromimage_raw_inverted_nm)

                                    print("greek&eng_raw_inverted_um")
                                    print(greekengtextfromimage_raw_inverted_um)

                            # ORDER OF PRIORITY IS, THRESHOLDED/BLURRED, CORRECT INVERSION(ATTEMPTED), OPPOSITE INVERSION.

                            # Criteria for determined text eligibility:
                            # First search for "um" (micron, since this gives less false positives) then "nm". Unit must be preceded by digit+space or digit to be considered.

                            digits = '.0123456789 '

                            if 'um' in greekengtextfromimage_edited_um and any(char in digits for char in greekengtextfromimage_edited_um):
                                if len(greekengtextfromimage_edited_um.split("um")[0]) > 0:
                                    if (greekengtextfromimage_edited_um.split("um")[0][-1] == " " or \
                                            greekengtextfromimage_edited_um.split("um")[0][-1].isdigit() == True):

                                        conversion = 10 ** -6
                                        if all(char in digits for char in greekengtextfromimage_edited_um.split("um")[0]):
                                            scalevalue = greekengtextfromimage_edited_um.split("um")[0]
                                            boxes = edited_um_boxes

                            if scalevalue == None and "nm" in greekengtextfromimage_edited_nm and any(char in digits for char in greekengtextfromimage_edited_nm):
                                if len(greekengtextfromimage_edited_nm.split("nm")[0]) > 0:
                                    if (greekengtextfromimage_edited_nm.split("nm")[0][-1] == " " or \
                                            greekengtextfromimage_edited_nm.split("nm")[0][-1].isdigit() == True):

                                        conversion = 10 ** -9
                                        if all(char in digits for char in greekengtextfromimage_edited_nm.split("nm")[0]):
                                            scalevalue = greekengtextfromimage_edited_nm.split("nm")[0]
                                            boxes = edited_nm_boxes

                            # Also check raw region only on the first pass just in case its legible.
                            elif pixel_value == 35:

                                if 'um' in greekengtextfromimage_raw_um and any(char in digits for char in greekengtextfromimage_raw_um):
                                    if len(greekengtextfromimage_raw_um.split("um")[0]) > 0:
                                        if ((greekengtextfromimage_raw_um.split("um")[0][-1] == " " and \
                                             greekengtextfromimage_raw_um.split("um")[0][-2].isdigit() == True) or \
                                                greekengtextfromimage_raw_um.split("um")[0][-1].isdigit() == True):

                                            conversion = 10 ** -6

                                            if all(char in digits for char in greekengtextfromimage_raw_um.split("um")[0]):
                                                scalevalue = greekengtextfromimage_raw_um.split("um")[0]
                                                boxes = raw_um_boxes

                                if scalevalue == None and "nm" in greekengtextfromimage_raw_nm and any(char in digits for char in greekengtextfromimage_raw_nm):
                                    if len(greekengtextfromimage_raw_nm.split("nm")[0]) > 0:
                                        if (greekengtextfromimage_raw_nm.split("nm")[0][-1] == " " or \
                                                greekengtextfromimage_raw_nm.split("nm")[0][-1].isdigit() == True):

                                            conversion = 10 ** -9

                                            if all(char in digits for char in greekengtextfromimage_raw_nm.split("nm")[0]):
                                                scalevalue = greekengtextfromimage_raw_nm.split("nm")[0]
                                                boxes = raw_nm_boxes

                                if scalevalue == None and 'um' in greekengtextfromimage_raw_inverted_um and any(char in digits for char in greekengtextfromimage_raw_inverted_um):
                                    if len(greekengtextfromimage_raw_inverted_um.split("um")[0]) > 0:
                                        if (greekengtextfromimage_raw_inverted_um.split("um")[0][-1] == " " or \
                                                greekengtextfromimage_raw_inverted_um.split("um")[0][-1].isdigit() == True):

                                            conversion = 10 ** -6

                                            if all(char in digits for char in greekengtextfromimage_raw_inverted_um.split("um")[0]):
                                                scalevalue = greekengtextfromimage_raw_inverted_um.split("um")[0]
                                                boxes = inverted_um_boxes

                                if scalevalue == None and "nm" in greekengtextfromimage_raw_inverted_nm and any(char in digits for char in greekengtextfromimage_raw_inverted_nm):
                                    if len(greekengtextfromimage_raw_inverted_nm.split("nm")[0]) > 0:
                                        if (greekengtextfromimage_raw_inverted_nm.split("nm")[0][-1] == " " or \
                                                greekengtextfromimage_raw_inverted_nm.split("nm")[0][-1].isdigit() == True):

                                            conversion = 10 ** -9

                                            if all(char in digits for char in greekengtextfromimage_raw_inverted_nm.split("nm")[0]):
                                                scalevalue = greekengtextfromimage_raw_inverted_nm.split("nm")[0]
                                                boxes = inverted_nm_boxes

                            # Filter out unrealistic scalevalues.
                            if scalevalue != None:
                                scalevalue = ''.join(i for i in scalevalue if i.isdigit())
                                try:
                                    if (float(scalevalue) < 0.01 or float(scalevalue) > 1000) or (float(scalevalue) > 10 and not int(scalevalue) % 5 == 0):
                                        scalevalue = None
                                except ValueError:
                                    scalevalue = None

                        # If both a scalevalue and a conversion that meet all criteria are found we can break out of the incremental thresholding of the region.
                        else:
                            break

            if conversion != None and scalevalue != None:
                scalevalue = float(''.join(i for i in scalevalue if i.isdigit()))
                potential_scale_bar_regions.append([i, conversion, scalevalue, is_box_in_box, boxes])
                if printer == True:
                    print("worked at pixel_value ", str(pixel_value + 5))  # since loop is stopped one loop too late
                    print("found eligible text, appended info")
                    print([i, conversion, scalevalue])
                    print("appended boxes ", boxes)
            else:
                boxes = None

        area_index += 1

    # Now that we have regions that include a number + unit, we need to search their greater regions for scale bars.
    # Since the bar will be somewhere near this text.

    potential_scale_bars = []
    for potential_region in potential_scale_bar_regions:
        region = potential_region[0]

        # If the region is a "box_in_box" we don't have to expand the search area as the bar will be within the same box.
        if potential_region[3] == True:
            region_border_buffer = 0

            regionsurroundingscalebar = gimg[region[1]:region[1] + region[3] + region_border_buffer, \
                                        region[0]:region[0] + region[2] + region_border_buffer]
            surrounding_region_y1 = region[0]
            surrounding_region_x1 = region[1]

        # If not box_in_box we expand our search region.
        else:
            surrounding_region_x1 = region[1] - region[3]

            if surrounding_region_x1 < 0:
                surrounding_region_x1 = 0

            surrounding_region_x2 = region[1] + 2 * region[3]

            if surrounding_region_x2 > rows:
                surrounding_region_x2 = rows

            surrounding_region_y1 = region[0] - region[2]

            if surrounding_region_y1 < 0:
                surrounding_region_y1 = 0

            surrounding_region_y2 = region[0] + 2 * region[2]

            if surrounding_region_y2 > cols:
                surrounding_region_y2 = cols

            regionsurroundingscalebar = thresholdedimg[surrounding_region_x1:surrounding_region_x2, \
                                        surrounding_region_y1:surrounding_region_y2]

            restrict_rectangles_color = None

        # Search the area for rectangles.
        h, w = regionsurroundingscalebar.shape[:2]
        regionsurroundingscalebar = cv2.resize(regionsurroundingscalebar, (w * interp_scale, h * interp_scale), interpolation=cv2.INTER_CUBIC)

        filteredvertices, boundingrectanglewidths, boundingrectanglecoords = find_draw_contours \
            (regionsurroundingscalebar, 151, minarea=10, nofilter=True, testing=show, annotate=show, bounding_rect=True, restrict_rectangles=True, restrict_rectangles_color=restrict_rectangles_color)

        # Quick filter on detected rectangle, w > h by at least 1.3 and width smaller than img width/1.5
        first_scale_bar_rectangles = [a for a in boundingrectanglecoords if float(a[2]) / a[3] >= 1.3 and a[2] < interp_scale*cols / 1.5]

        potential_scale_bar_rectangles = []

        h, w = gimg.shape[:2]
        gimg_for_fsb = cv2.resize(gimg, (w * interp_scale, h * interp_scale), interpolation=cv2.INTER_CUBIC)

        # Checks of whether  scale bars of solid color. (Not a chunk of the image by mistake)
        for i in first_scale_bar_rectangles:

            fsb = gimg_for_fsb[i[1] + interp_scale*surrounding_region_x1:i[1] + interp_scale*surrounding_region_x1 + i[3], i[0] + interp_scale*surrounding_region_y1:i[0] +interp_scale* surrounding_region_y1 + i[2]].flatten()

            fsb_stdev = np.std(fsb)
            fsb_mean = sum(fsb) / len(fsb)
            fsb_mode, _ = stats.mode(fsb)
            fsb_mode = fsb_mode[0]

            # uniform color that's either black or white.
            if fsb_mean > 200 or fsb_mean < 30 or fsb_mode > 245 or fsb_mode < 10:
                potential_scale_bar_rectangles.append(i)

        scalebarboundingrectangle = None
        scalebarwidth = None

        potential_scale_bar_rectangles = [np.array(a) // interp_scale for a in potential_scale_bar_rectangles]

        # If we still have multiple candidates in this region, we just go with the widest one.
        potential_scale_bar_widths = [a[2] for a in potential_scale_bar_rectangles]

        if len(potential_scale_bar_widths) > 0:
            scalebarwidth = max(potential_scale_bar_widths)
            scalebarboundingrectangle = potential_scale_bar_rectangles[potential_scale_bar_widths.index(scalebarwidth)]

            boxes_relative = potential_region[-1]
            boxes = []

            for b in boxes_relative.splitlines():
                b = b.split(' ')

                if str(b[0]) in "0123456789num":
                    b = [int(i) // interp_scale for i in b[1:5]]

                    boxes.append([(b[0] + region[0], region[3] - b[1] + region[1]), (b[2] + region[0], region[3] - b[3] + region[1])])

                if str(b[0]) == "m":
                    break

            potential_scale_bars.append([(int(scalebarboundingrectangle[0] + surrounding_region_y1), int(scalebarboundingrectangle[1] + surrounding_region_x1) \
                                              , int(scalebarboundingrectangle[2]), int(scalebarboundingrectangle[3])), potential_region[1], potential_region[2], boxes])

    # If we have multiple candidates, in multiple regions of interest, we go with the widest one globally.
    widths = [a[0][2] for a in potential_scale_bars]

    if len(widths) > 0:
        scalebarwidth = max(widths)
        scalebar, conversion, scalevalue, boxes = potential_scale_bars[widths.index(max(widths))]

    if all(i is not None for i in [scalebar, conversion, scalevalue, boxes]):

        ##more sophisticated inlay construction, going to pass image_to_boxes during reading and use it here.
        # using bounding boxes of detected characters + scalebar to determine accurate inlay location.

        box_xs = []
        box_ys = []

        for box in boxes:
            box_xs.append(box[0][0])
            box_xs.append(box[1][0])

            box_ys.append(box[0][1])
            box_ys.append(box[1][1])

        box_xs.append(scalebar[0])
        box_xs.append(scalebar[0] + scalebar[2])

        box_ys.append(scalebar[1])
        box_ys.append(scalebar[1] + scalebar[3])

        inlaycoords.append((min(box_xs), min(box_ys), max(box_xs) - min(box_xs), max(box_ys) - min(box_ys)))

    return scalebar, scalevalue, conversion, inlaycoords


import random
import sys
import time
from lifegame import LifeGame

def init():
    base_x = 25
    base_y = 25
    game = LifeGame(base_x, base_y)
    for x in range(base_x):
        for y in range(base_y):
            game.set_alive(y, x, random.randint(0, 1))
    return game

def console_out(game):
    sys.stdout.write("\r")
    for y in range(game.y):
        for x in range(game.x):
            sys.stdout.write("■ " if game.board[y][x] else "□ ")
        sys.stdout.write("\n")
    sys.stdout.flush()
    print()
    time.sleep(1)

def main():
    game = init()
    while(True):
        console_out(game)
        game.next_generation()

if __name__ == "__main__":
    main()


import requests
from bs4 import BeautifulSoup
from flask import Flask, render_template, request

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/scrape', methods=['POST'])
def scrape():
    url = request.form['url']
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    links = soup.find_all('a')
    scraped_links = [link.get('href') for link in links]
    return render_template('results.html', links=scraped_links)

if __name__ == '__main__':
    app.run(debug=True)


"""Automated embedding of resources in Python Packages/Modules

ResourcePackage provides the Python programmer with a
mechanism for automatically maintaining a collection of
resources embedded in Python source-code modules.

The basic approach taken is to embed one resource in
each python module of a package explicitly intended
to be a resource-only package.

See the doc sub-directory for documentation
"""
__version__ = (1,0,1)
__author__ = "Michael Colin Fletcher"
__license__ = "BSD-Style, see license.txt for details"


import sys
import operator
class Solution(object):
    def findMinArrowShots(self, points):
        """
        :type points: List[List[int]]
        :rtype: int
        """
        if len(points) < 2:
            return len(points)

        points.sort(key=operator.itemgetter(0))
        # stores number of arrows to be shot. You will atleast shoot 1 arrow
        # why? because <2 length of points has already been checked
        # so there are atleast 2 points, so you'll need at least 1 arrow
        res = 1
        minend = sys.maxsize
        for p in range(0, len(points)):
            curr = points[p]
            # if start of current balloon is more than minend, then a new cluster is made
            # so shoot an arrow to eliminate the old cluster, ie, increment res
            if curr[0] > minend:
                res +=1 # shoot at the minend to kill the entire cluster
                minend = curr[1] # update new cluster's end as minend
            else:
                # there is an overlap, curr belongs to same active cluster
                # this new balloon "curr" can be handled by the same arrow
                # when we kill the other previous balloons in the cluster
                # we're capped by the min of the end of all balloons in the cluster as
                # it is at pos "minend" where we will shoot the arrow to kill the cluster
                minend = min(minend, curr[1])
        return res


import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import glob
import json

def meta_dataframe(path):
    """
    Load the metadata of the dateset.

    'title' and 'journal' attributes may be useful later when we cluster
    the articles to see what kinds of articles cluster together.
    """
    return pd.read_csv(path, dtype = {
        'pubmed_id': str,
        'Microsoft Academic Paper ID': str,
        'doi': str
    })

class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
        self.paper_id = content['paper_id']
        self.abstract = [entry['text'] for entry in content['abstract']]
        self.body_text = [entry['text'] for entry in content['body_text']]
        self.abstract = '\n'.join(self.abstract)
        self.body_text = '\n'.join(self.body_text)
    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'

def get_breaks(content, length):
    """
    Add an html break every n words.

    Ensures the hover tool of the interactive plot fits the screen.
    """
    data = ""
    words = content.split(' ')
    total_chars = 0

    # add break every length characters
    for i in range(len(words)):
        total_chars += len(words[i])
        if total_chars > length:
            data = data + "<br>" + words[i]
            total_chars = 0
        else:
            data = data + " " + words[i]
    return data

def json_paths(root_path):
    """
    Fetch path to all JSON files.
    """
    return glob.glob(f'{root_path}/**/*.json', recursive = True)

def paper_dataframe(
    json_paths,
    abstract_snippet_wordcount = 100,
    line_length = 40
):
    """
    Reads json-formatted articles into a Pandas DataFrame.
    """

    data = {
        'paper_id': [],
        'doi' : [],
        'abstract' : [],
        'body_text' : [],
        'authors' : [],
        'title' : [],
        'journal' : [],
        'abstract_summary' : []
    }

    for index, entry in enumerate(json_paths):
        if index % (len(json_paths) // 10) == 0:
            print(f'Processing index: {index} of {len(json_paths)}')

        try: content = FileReader(entry)
        except: continue  # invalid paper format, skip

        meta_dataframe = meta_dataframe(json_paths)
        meta_data = meta_dataframe.loc[meta_df['sha'] == content.paper_id]
        if len(meta_data) == 0: continue # no metadata, skip this paper

        data['abstract'].append(content.abstract)
        data['paper_id'].append(content.paper_id)
        data['body_text'].append(content.body_text)

        # create a column for an abstract snippet to be used in a plot
        if len(content.abstract) == 0:
            data['abstract_summary'].append("Not provided.")
        elif len(content.abstract.split(' ')) > abstract_snippet_wordcount:
            info = content.abstract.split(' ')[:abstract_snippet_wordcount]
            summary = get_breaks(' '.join(info), line_length)
            data['abstract_summary'].append(summary + "...")
        else:
            summary = get_breaks(content.abstract, line_length)
            data['abstract_summary'].append(summary)

        try:
            authors = meta_data['authors'].values[0].split(';')
            if len(authors) > 2:
                columns['authors'].append(
                    get_breaks('. '.join(authors), line_length)
                )
            else: data['authors'].append(". ".join(authors))
        except: # if only one author - or Null value
            data['authors'].append(meta_data['authors'].values[0])

        try: # add the title information, add breaks when needed
            title = get_breaks(meta_data['title'].values[0], line_length)
            data['title'].append(title)
        except: # if title was not provided
            data['title'].append(meta_data['title'].values[0])

        data['journal'].append(meta_data['journal'].values[0])
        data['doi'].append(meta_data['doi'].values[0])

        return pd.DataFrame(data, columns = data.keys)


from sympy import symbols, var, simplify, solveset
var('x0,i,k,ts,kc')

Aanalytic =  (x0 - i/k)*1/k*(1-exp(-k*ts)) + i/k*ts
Alinear = ts*(x0 - ts/2*(kc*x0 - i))

s = solveset(Eq(Aanalytic,Alinear), kc)

str(s)
print(latex(s))


import os
from PIL import Image, ImageDraw, ImageFont
from uuid import uuid4

path = os.path.dirname(os.path.abspath(__file__))

class Wordle:
    def __init__(self, word, chat_id, user_id, word_url):
        self.word = word.upper()
        self.word_url = word_url
        self._telegram_id = ""
        self.chat_id = chat_id
        self.user_id = user_id
        self.level = 0
        self.filename = "tiles-container.webp"
        self._board = Image.open(f"{path}/tiles-container.jpg")
        self._correct = Image.open(f"{path}/correct-background.jpg").resize(
            (43, 43), Image.LANCZOS
        )
        self._wrong = Image.open(f"{path}/wrong-background.jpg").resize(
            (43, 43), Image.LANCZOS
        )
        self._partial = Image.open(f"{path}/partially-correct-background.jpg").resize(
            (43, 43), Image.LANCZOS
        )
        self._font = ImageFont.truetype(f"{path}/Roboto-Regular.ttf", size=42)
        self._y = 11

    def verify_guess(self, guess):
        self._x = 11
        guess = guess.upper()
        if guess == self.word:
            for letter in self.word:
                self._board.paste(self._correct, (self._x, self._y))
                draw = ImageDraw.Draw(self._board)
                draw.text(
                    (self._x + 8, self._y - 2),
                    text=letter,
                    fill="white",
                    font=self._font,
                )
                self._x += 59
            self.filename = f"{str(uuid4())}.webp"
            self._board.save(f"{path}/{self.filename}")
            return True

        else:
            for index, letter in enumerate(guess):
                if letter == self.word[index]:
                    self._board.paste(self._correct, (self._x, self._y))
                elif letter in self.word and letter != self.word[index]:
                    self._board.paste(self._partial, (self._x, self._y))
                else:
                    self._board.paste(self._wrong, (self._x, self._y))
                draw = ImageDraw.Draw(self._board)
                draw.text(
                    (self._x + 8, self._y - 2),
                    text=letter,
                    fill="white",
                    font=self._font,
                )
                self._x += 59
        self._y += 58
        self.filename = f"{str(uuid4())}.webp"
        self._board.save(f"{path}/{self.filename}")
        self.level += 1
        return False


# -*- coding: utf-8 -*-

from flask import Blueprint, jsonify, request
from flask.views import MethodView
from mongoengine import NotUniqueError
from mongoengine.queryset import DoesNotExist

from .cors import cors
from .models import Device

devices = Blueprint('devices', __name__)

class RequestMalformedError(Exception):
    pass

class DevicesView(MethodView):

    @cors()
    def get(self):
        ids = request.args.getlist('ids[]')
        if len(ids) != 0:
            rdevices = Device.objects(device_id__in=ids)
        else:
            rdevices = Device.objects()

        filtered_query = Device.objects.translate_to_jsonable(request.args)
        rdevices = rdevices(**filtered_query)

        return jsonify({'devices': rdevices.to_jsonable()})

    @cors()
    def post(self):
        device_dict = request.json.get('device', None)
        if device_dict is None:
            raise RequestMalformedError

        vk_pem = device_dict.get('vk_pem', None)
        if vk_pem is None:
            raise RequestMalformedError

        d = Device.create(vk_pem)
        return jsonify({'device': d.to_jsonable()}), 201

    @cors()
    def options(self):
        pass

devices.add_url_rule('', view_func=DevicesView.as_view('devices'))

class DeviceView(MethodView):

    @cors()
    def get(self, device_id):
        d = Device.objects.get(device_id=device_id)
        return jsonify({'device': d.to_jsonable()})

    @cors()
    def options(self):
        pass

devices.add_url_rule('/<device_id>',
                     view_func=DeviceView.as_view('device'))

@devices.errorhandler(400)
@devices.errorhandler(RequestMalformedError)
@cors()
def malformed(error):
    return jsonify(
        {'error': {'status_code': 400,
                   'type': 'Malformed',
                   'message': 'Request body is malformed'}}), 400

@devices.errorhandler(NotUniqueError)
@cors()
def not_unique_error(error):
    return jsonify(
        {'error': {'status_code': 409,
                   'type': 'FieldConflict',
                   'message': 'The value is already taken'}}), 409

@devices.errorhandler(DoesNotExist)
@cors()
def does_not_exist(error):
    return jsonify(
        {'error': {'status_code': 404,
                   'type': 'DoesNotExist',
                   'message': 'Item does not exist'}}), 404


from chatterbot.treiners import ListTrainer

from chatterbot import ChatBot
from chatterbot import corpus

bot = ChatBot('T bot')

conversa =['Oi', 'Ola', 'Tudo bem?', 'Tudo ótimo', 'Como você esta?' ,'Bem e você?']

bot.set_Trainer(ListTrainer)
bot.train(conversa)

while True:
    pergunta= input('User :')
    resposta = bot.get_response(pergunta)
    if float(resposta.confidence)> 0.5:
        print('T Bot: ', resposta)
    else:
        print('T Bot: ainda não sei responder esta pergunta')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  probability.py

from math import log

def probabilities (X) -> dict:
	""" This function maps the set of outcomes found in the sequence of events, 'X', to their respective probabilty of occuring in 'X'.
	The return value is a python dictionary where the keys are the set of outcomes and the values are their associated probabilities."""
	# The set of outcomes, denoted as 'C', and the total events, denoted as 'T'.
	C, T = set(X), len(X)
	return {c: X.count(c) / T for c in C}

def shannon (P, b : int = None) -> float:
	""" Shannon or Informational Entropy
	Calculates the shannon entropy, denoted as 'H', of a discrete random variable, 'X'.
	Parameters:
	1.) P:  An iterable of probabilities associated with the set of outcomes of 'X'.
	2.) b:  The logarithmic base used in calculating 'H'.  The default value is len(P). """
	# Check params.
	if b == None: b = len(P)
	if b <= 0: raise ValueError('Argument Value Error:\tThe logarithmic base must be a positive integer.')
	# Init return value.
	res = 0
	for p in P:
		res += p*log(p)
	res /= log(b)
	return -res

def main(args):

    return 0

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv))


import numpy as np

#Problem #3 Encryption

encriptList = [8, 15, 23, 1, 18, 5, 25, 15, 21]
moduloN = 26
encryptedList = []
for x in range(len(encriptList)):
    encryptedList.append((encriptList[x]**5) % moduloN)

# print(encryptedList)

affineList = [9, 5, 1, 20, 3, 8, 5, 5, 19, 5]
affineEncrypted = []
for x in range(len(affineList)):
    affineEncrypted.append((affineList[x]*3 + 8) % moduloN)

# print(affineEncrypted)

for x in range(1,27):
    print(f"Term: {x}\tExponentiated: {(x**5)%26}")


class Solution:
    def sortString(self, s: str) -> str:
        s = sorted(s)
        hashmap = {}
        for i in s:
            if i in hashmap:hashmap[i]+=1
            else:hashmap[i]=1
        newStr = []
        while hashmap:
            print(hashmap)
            keys = list(hashmap.keys()) # Create a copy of the dictionary keys
            revStr = ""
            for k in keys:
                if hashmap[k]>1:
                    revStr+=k
                    hashmap[k]-=2
                    if hashmap[k]==0:del hashmap[k]
                elif hashmap[k]<=1:del hashmap[k]
            newStr+=keys
            newStr+=revStr[::-1]
        return "".join(newStr)


class Aluno():
    def __init__(self, nome, idade, matricula):
        self.nome = nome
        self.idade = idade
        self.matricula = matricula

nome = input('insira o nome: ')
idade = input('insira a idade: ')
matricula = input('insira o numero da matricula: ')
aluno = Aluno(nome,idade,matricula)
print(aluno.nome, aluno.idade, aluno.matricula)

'''class Professor(Aluno):
  def __init__(self, formacao, turma):
    self.formacao = formacao
    self.turma = turma

  Professor = input('insira o nome do professor: ')
  idade = input('insira a idade do professor: ')
  matricula = input('insira o número da maricula do professor: ')
  professor = Professor(Professor, nome, idade, matricula)
  print(professor.nome, professor.idade, professor.matricula)'''


# time for some fun
from ghidra.app.plugin.core.navigation.locationreferences import ReferenceUtils
from ghidra.program.model.symbol import SourceType

s_text = getMemoryBlock('.text')
s_init_array = getMemoryBlock('.init_array')
s_rodata = getMemoryBlock('.rodata.1') # not sure if it's .1 in all bins?
s_data = getMemoryBlock('.data')

def get_all_inits():
	addr = s_init_array.start
	while addr < s_init_array.end:
		yield getDataAt(addr).value
		addr = addr.add(8)

def do_all_inits():
	num = 0
	for ptr in get_all_inits():
		if getInstructionAt(ptr) is None:
			disassemble(ptr)
		if getFunctionAt(ptr) is None:
			createFunction(ptr, None)
		fn = getFunctionAt(ptr)
		if fn.name.startswith('FUN_'):
			fn.setName('Init%04d_130' % num, ghidra.program.model.symbol.SourceType.DEFAULT)
		num += 1

def remove_all_spurious_data():
	addr = s_text.start
	while addr < s_text.end:
		removeDataAt(addr)
		addr = addr.add(1)

def nops_following_ret(addr):
	if getInt(addr.subtract(4))&0xFFFFFFFF == 0xd65f03c0:
		while (addr.offset & 0xF) != 0:
			if getInt(addr)&0xFFFFFFFF != 0xd503201f:
				return False
			addr = addr.add(4)
		return True
	return False

# assume that the first fn is defined, probably at 7100000030
def locate_undefined_candidates(start):
	addr = start
	while addr < s_text.end:
		oaddr = addr
		c1 = getUndefinedDataAfter(addr).address
		c2 = getDataAfter(addr).address
		addr = c1 if (c1 < c2) else c2
		if addr >= s_text.end:
			break
		if (addr.offset & 0xF) != 0:
			# not aligned
			addrval = getInt(addr)&0xFFFFFFFF
			if addrval == 0xd61f0140:
				# br x10, for switch tables Ghidra fails to identify
				disassemble(addr)
				continue
			elif addrval == 0 or addrval == 0xE7FFDEFE or nops_following_ret(addr):
				addr = addr.add(0x10 - (addr.offset & 0xF))
			else:
				#print('found unaligned non-zeroes @ %r, what' % addr)
				#print('our old addr was %r, c1=%r, c2=%r' % (oaddr, c1, c2))
				#return oaddr
				disassemble(addr)

		if getInt(addr) == 0:
			print('found aligned zeroes @ %r' % addr)
			return
		else:
			if getInstructionAt(addr) == None:
				print('possible code @ %r' % addr)
				if getInt(addr)&0xFFFFFFFF == 0xE7FFDEFE:
					addr = addr.add(4)
				else:
					disassemble(addr)

def force_eliminate_data(addr, count):
	for i in xrange(count):
		d = getDataContaining(addr.add(i))
		if d is not None:
			removeData(d)

def is_pointer_at_addr(addr):
	data = getDataAt(addr)
	if data is None:
		return False
	else:
		return data.isPointer()

def scan_data_for_code_ptrs():
	found = 0
	addr = s_data.start
	end = s_data.end.subtract(7)
	ts, te = s_text.start.offset, s_text.end.offset
	while addr < end:
		val = getLong(addr)
		if val >= ts and val < te and not is_pointer_at_addr(addr):
			print('unreffed pointer into .text @ %r -> %x' % (addr, val))
			clearListing(addr, addr.add(7))
			createData(addr, ghidra.program.model.data.Pointer64DataType())
			found += 1
			#if found > 20: break
		addr = addr.add(8)

def findEvFlowVtable(ctor):
	body = getFunctionAt(ctor).body
	candidates = []
	for addr in body.getAddresses(True):
		for ref in getReferencesFrom(addr):
			rt = ref.referenceType
			if rt.data and not rt.read and not rt.write and ref.operandIndex == 0 and ref.isMemoryReference() and getDataAt(ref.toAddress).isPointer():
				return ref.toAddress

def find_init(addr):
	for check in inits:
		if addr < check:
			return check

vtables_to_names = {}
#missinga = open('/Users/ash/src/hashcat/missing_actions', 'w')
#missingq = open('/Users/ash/src/hashcat/missing_queries', 'w')

def name_eventflow_classes(path):
	'''import json
	with open(path, 'r') as f:
		data = json.load(f)
	queries, actions = {}, {}
	for ac in data.values():
		for q,k in ac['queries'].iteritems(): queries[k] = q
		for a,k in ac['actions'].iteritems(): actions[k] = a'''
	path = '/Users/ash/src/hashcat'
	queries, actions = {}, {}
	for line in open(path+'/actions_cracked', 'r'):
		line = line.strip()
		if line:
			a,b,c = line.split(':')
			actions[int(a,16)] = c
	for line in open(path+'/queries_cracked', 'r'):
		line = line.strip()
		if line:
			a,b,c = line.split(':')
			queries[int(a,16)] = c

	for names,baseaddr,what in ((queries,'eventFlowQueryMap','Query'), (actions,'eventFlowActionMap','Action')):
		in_exec = set()
		missing = set()
		qmap = getDataAt(toAddr(baseaddr))
		count = qmap.length//16
		#count = 3
		for i in xrange(count):
			key = getInt(qmap.address.add(i * 16)) & 0xFFFFFFFF
			if key in names:
				#print('found %s' % names[key])
				in_exec.add(key)
				name = names[key]
			else:
				#print('missing %08x!' % key)
				missing.add(key)
				name = 'EventFlow%s%08X' % (what, key)
				if what == 'Query':
					missingq.write('%08x:00000000\n' % key)
				else:
					missinga.write('%08x:00000000\n' % key)
			addr = toAddr(getLong(qmap.address.add(i * 16 + 8)))
			#getFunctionAt(addr).setName(name + '_build', ghidra.program.model.symbol.SourceType.DEFAULT)

			# find ctor
			ctor = getReferencesFrom(addr.add(0x24))[0].toAddress
			#print('ctor %s = %r' % (name, ctor))
			#getFunctionAt(ctor).setName(name + '_ctor', ghidra.program.model.symbol.SourceType.DEFAULT)

			# find vtable
			vt = findEvFlowVtable(ctor)
			#createLabel(vt, name+'_vtable', True)
			vtables_to_names[vt.offset] = name
			vtsym = getSymbolAt(vt)
			vtns = vtsym.parentNamespace
			#print('vt %r is %r' % (name,vtns))

			if vtns.name != name:
				print('would rename %r to %r' % (vtns,name))
				vtns.symbol.setName(name, SourceType.ANALYSIS)

			init = find_init(ctor)
			#print('%r goes with %r' % (vtns, getFunctionAt(init)))
			initmap[init].append(vtns)

		#in_json = set(names.keys())
		#noted = in_json - in_exec
		#for n in noted:
		#	print('missing %s from exec table but in json: %s' % (what, names[n]))

# STEP 1: mark all init functions
#do_all_inits()
# STEP 2: remove all spurious data from the .text section
#remove_all_spurious_data()
# STEP 3: disassemble all code
#locate_undefined_candidates(getFirstFunction().entryPoint)
# STEP 4: set all code ptrs (vtables, etc) to actual references
scan_data_for_code_ptrs()
# STEP 5: use the FindInstructionsNotInsideFunctionScript and CreateFunctionsFromSelection scripts

'''
inits = list(get_all_inits())
addr = s_init_array.start
for i in xrange(len(inits) - 1):
	a = inits[i]
	b = inits[i + 1]
	delta = getFunctionAt(b).body.minAddress.subtract(getFunctionAt(a).body.maxAddress) - 1
	if delta < 0x10:
		setEOLComment(addr.add(8 + i * 8), '-')
	else:
		setEOLComment(addr.add(8 + i * 8), '0x%x' % delta)

initmap = {}
for init in inits:
	initmap[init] = []
'''

# STEP 6: set up some eventflow stuff
#name_eventflow_classes('/Volumes/HFS/repos/switch/CylindricalEarth/evfl/evflActors120.json')
#import json
#with open('/Volumes/HFS/repos/switch/CylindricalEarth/evfl/eventflow_vtables.json', 'w') as f:
#	json.dump(vtables_to_names, f)

'''
for init in inits:
	if len(initmap[init]) > 1:
		print(getFunctionAt(init))
		for i in initmap[init]:
			print(i)
	elif len(initmap[init]) == 1:
		ns = initmap[init][0]
		initsym = getSymbolAt(init)
		if initsym.name != 'STATIC_INIT_':
			initsym.setNameAndNamespace('STATIC_INIT_', ns, SourceType.USER_DEFINED)
'''

#missingq.close()
#missinga.close()

#addr = getFirstFunction().entryPoint
#while True:
#	new_addr = locate_undefined_candidates(getFirstFunction().entryPoint)
#	if new_addr is None:
#		print('all done?')
#		break
#	if new_addr == addr:
#		print('might\'ve fucked it')
#		break
#	addr = new_addr
#	print('wait...')
#	analyzeChanges(currentProgram)


## map sequences into color space, translating pairwise sequence distances into perceptual difference of colors

## sequence distance can be measured as edit-distance between the two.
## perceptual difference of colors is a topic extensively studied, but here, simply, CIELAB is used

## TODO: organize the usage
## param: mode=

from skimage.color import lab2rgb
import numpy  as np
import sys, re

def openFasta(path):
    """ open fasta as simple dict """

    from Bio.SeqIO import FastaIO
    with open(path) as handle:
        return dict(FastaIO.SimpleFastaParser(handle))
        #return dict(list(FastaIO.SimpleFastaParser(handle))[:30])

# TODO: export this
def seq_to_edit_distance(seq_dict):
    """ pairwise edit distance is calculated from dict of type {seqname, seq}"""

    from Bio import pairwise2
    #dim =  len(list(seq_dict.keys()))
    dim =  len(seq_dict.keys())
    d = np.zeros((dim, dim), dtype=np.int)
    s = list(seq_dict.values())

    for i in range(dim):
        print(i)
        sys.stdout.flush()

        for j in range(dim):
            if i == j:
                d[i,j] = 0 # put 0
            elif i > j:
                d[i,j] = d[j,i] # put d[i][j] = d[j][i]
            else:
                score = pairwise2.align.globalms(s[i], s[j], 0, -1, -1, -1, score_only = True) # calc d[i][j]
                d[i,j] = -score
    return d

# TODO: export this
# by default it returns [0,255]*3 RGB
def distMat2colors(dm, lab = False):

    # tsne into 3dim
    from sklearn.manifold import TSNE
    dm_3dim = TSNE(n_components=3, metric="precomputed").fit_transform(dm)
    n = dm_3dim.shape[0]
    #print(dm_3dim)

    # embedding is transformed to fit in CIELAB
    from skimage.color import lab2rgb

    #rgb1 = lab2rgb(dm_3dim.reshape(n, 1, 3))
    #print(rgb1.reshape(n, 3))

    #b, t = dm_3dim.min(0), dm_3dim.max(0)
    b, t = dm_3dim.min(), dm_3dim.max()
    #valid_lab = (dm_3dim - b) * np.array([99.99, 254.99, 254.99]) / (t - b) - np.array([0, 127, 127]) # range of value = 100, +-127, +-127
    valid_lab = (dm_3dim - b) * np.array([100.0, 100, 100]) / (t - b) # percentage ??
    #valid_lab = (np.array([1, 2, 2]) * (dm_3dim - b) / (t - b)) - np.array([0, 1, 1]) # range of value = [0,1] for each

    if lab:
        # return CIELAB for each elem
        # return valid_lab
        print("lab[0-1]")
        print(valid_lab)
        print("3dim")
        print(dm_3dim)
        return valid_lab
    else:
        # return RGB
        rgb2 = lab2rgb(valid_lab.reshape(n, 1, 3)).reshape(n, 3)
        print("rgb2")
        print(rgb2)
        print("rgb2[0-255]")
        print(rgb2 * 256)
        return rgb2 * 256

path = "./data/monomers/d0.fa"
#path = "./data/monomers/MigaKH.HigherOrderRptMon.fa"
seq_dict = openFasta(path)
names, sequences = list(seq_dict.keys()), list(seq_dict.values())

# dm = np.load("./count_vars/monomers.dist.npy")
dm = seq_to_edit_distance(seq_dict)

cols = distMat2colors(dm)
with open(re.sub(".*/", "", f"{path}") + ".colors", "w") as f:
    for i in range(len(names)):
        f.write(f"{names[i]}\t{cols[i,0]}\t{cols[i,1]}\t{cols[i,2]}\n")

cols = distMat2colors(dm, lab=True)
with open(re.sub(".*/", "", f"{path}") + ".colors.lab", "w") as f:
    for i in range(len(names)):
        f.write(f"{names[i]}\t{cols[i,0]}\t{cols[i,1]}\t{cols[i,2]}\n")


import plenty.models as models
from decimal import Decimal
from datetime import datetime, timedelta

async def token_price_change_percentage_minutes(symbol_token, price_token_dollar, amount_minutes):
    timestamp_now = datetime.utcnow()
    timestamp_minutes_ago = timestamp_now - timedelta(minutes=amount_minutes)

    token_day_stats = await models.TokenStats \
        .filter(symbol_token=symbol_token) \
        .filter(timestamp__gte=timestamp_minutes_ago) \
        .filter(timestamp__lte=timestamp_now) \
        .order_by("timestamp") \
        .first()

    try:
        price_token_minutes_ago = token_day_stats.token_price
        price_change_token = Decimal(((price_token_dollar - price_token_minutes_ago) / price_token_minutes_ago) * 100)
    except (TypeError, AttributeError):
        price_change_token = Decimal(1)

    return price_change_token


# Execution variables
import os
class CACHE():
    def __init__(self, FILEPATH):
        self.FILEPATH = FILEPATH

    def update_path(self, FILEPATH):
        self.FILEPATH = FILEPATH

DATA_CACHE = CACHE(os.path.join(os.getcwd(), "CACHE"))
LOG_FILEPATH = os.path.join(DATA_CACHE.FILEPATH, "nemglo_sim.log")

# Naming conventions
variable_names = ['vre_cap']
constraint_names = []

cost_variables_shadow = ['impact_emissions', 'relief_prd_tgt', 'msl_penalise', 'msl_relieve']
cost_variables_realized = ['h2e_load', 'h2_production', 'ppa_vre']

# Electrolyser Default SEC variable profiles and characteristics
# Beerbühl et.al, 2015, https://doi.org/10.1016/j.ejor.2014.08.039
SEC_PROFILE_AE = {'h2e_load_pct': [0, 0.2, 0.4, 0.6, 0.8, 1.0],
                    'nominal_sec_pct': [0.8068, 0.8068, 0.8181, 0.8523, 0.9091, 1.0]}
# Kopp et.al, 2017, https://doi.org/10.1016/j.ijhydene.2016.12.145
SEC_PROFILE_PEM = {'h2e_load_pct': [0, 0.2, 0.4, 0.6, 0.8, 1.0],
                    'nominal_sec_pct': [0.8383, 0.8383, 0.8598, 0.8982, 0.9315, 1.0101]}
SEC_NOMINAL_AE = 50
SEC_NOMINAL_PEM = 66
SEC_CONVERSION_AE = 1.0
SEC_CONVERSION_PEM = 1.0

PALETTE = ['#b2d4ee','#849db1','#4f6980',
           '#B4E3BC','#89AE8F','#638b66',
           '#ffb04f','#de9945','#af7635',
           '#ff7371','#d6635f','#b65551',
           '#AD134C','#cc688d','#ff82b0']


import sys
import pandas as pd
import numpy as np
import os
import pickle
from sqlalchemy import create_engine
import re
import nltk
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import train_test_split
from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier,AdaBoostClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer, accuracy_score, f1_score, fbeta_score, classification_report
from nltk.corpus import stopwords
from scipy.stats import gmean

nltk.download(['punkt', 'wordnet', 'averaged_perceptron_tagger'])
nltk.download('stopwords')

def load_data(database_filepath):
    '''
    Input-: The database file info is provided

    Loads data from db table to this python script

    Output-:X,Y and columns of the Y

    '''

    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql_table('new_disaster',engine)
    df['related']=df['related'].map(lambda x: 1 if x == 2 else x)
    X = df['message']
    Y = df.iloc[:,4:]

    return X,Y,Y.columns

def tokenize(text,url_place_holder_string="urlplaceholder"):
    '''
    Input -: text

    Tokenize,lemmaize,and remove the url from the messages

    output - tokens in form of list

    '''

    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)

    for detected_url in detected_urls:
        text = text.replace(detected_url, url_place_holder_string)

    tokens = nltk.word_tokenize(text)
    lemmatizer = nltk.WordNetLemmatizer()

    # List of clean tokens
    clean_tokens = [lemmatizer.lemmatize(w).lower().strip() for w in tokens]
    return clean_tokens

class StartingVerbExtractor(BaseEstimator, TransformerMixin):
    """
    Starting Verb Extractor class

    This class extract the starting verb of a sentence,
    creating a new feature for the ML classifier
    """

    def starting_verb(self, text):
        sentence_list = nltk.sent_tokenize(text)
        for sentence in sentence_list:
            pos_tags = nltk.pos_tag(tokenize(sentence))
            first_word, first_tag = pos_tags[0]
            if first_tag in ['VB', 'VBP'] or first_word == 'RT':
                return True
        return False

    # Given it is a tranformer we can return the self
    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X_tagged = pd.Series(X).apply(self.starting_verb)
        return pd.DataFrame(X_tagged)

def build_model():
    '''
    This create pipeline which will be used for training prupose.

    Output-:Ml model

    '''
    pipeline = Pipeline([
        ('features', FeatureUnion([

            ('text_pipeline', Pipeline([
                ('count_vectorizer', CountVectorizer(tokenizer=tokenize)),
                ('tfidf_transformer', TfidfTransformer())
            ])),

            ('starting_verb_transformer', StartingVerbExtractor())
        ])),

        ('classifier', MultiOutputClassifier(AdaBoostClassifier()))
        ])

    parameters_grid = {#'classifier__estimator__learning_rate': [0.01, 0.02, 0.05],
              'classifier__estimator__n_estimators': [10, 20, 40],
            'features__transformer_weights': (
            {'text_pipeline': 1, 'starting_verb': 0.5},
            {'text_pipeline': 0.5, 'starting_verb': 1},
            {'text_pipeline': 0.8, 'starting_verb': 1},
    )}

    cv = GridSearchCV(pipeline, param_grid=parameters_grid, scoring='f1_micro', n_jobs=-1)

    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    '''
    Input-: we used test data to evaluate out trained model

    output-: report of individual columns with avg score

    '''

    y_prediction_test = model.predict(X_test)
    print("This is evalution of the training data set it has f1-score,precison,recall of all the columns")
    print(classification_report(Y_test.values, y_prediction_test, target_names=category_names.values))

def save_model(model, model_filepath):
    '''
    saved model as pickle file

    '''

    pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
filename1='C:\\Users\\guojia\\Desktop\\comment.csv'
filename2='C:\\Users\\guojia\\Desktop\\comment_1.csv'
data1=pd.read_csv(filename1,encoding='gbk',header=None)
data2=pd.read_csv(filename2,encoding='gbk',header=None)
data=pd.concat([data1,data2])
data=data.drop_duplicates()#删除重复项
data.dropna()#删除空值
import re
data_1=re.compile(r'[\u4e00-\u9fa5]+')#正则表达式匹配汉字，删除评论中的各种符号
data[0]=data[0].map(lambda x:''.join(re.findall(data_1,x)))
data_2=[]
for i in data[0]:
    line=i.replace('举报楼打赏回复评论赞','').replace('举报','').replace('评论','').replace('客户端','').replace('来自天涯社区','').replace('楼主','').replace('回复','').replace('作者','')
    data_2.append(line)
data_2=pd.DataFrame(data_2)
import jieba
mycut=lambda s:' '.join(jieba.cut(s,cut_all = False))
data3=data_2[0].apply(mycut)
outputfile='C:\\Users\\guojia\\Desktop\\output.txt'
data3.to_csv(outputfile,index=False,header=False,encoding='utf-8')
negfile='C:\\Users\\guojia\\Desktop\\output.txt'
stoplist='G:\\python_work\\stoplist.txt'
data_n=pd.read_csv(negfile,encoding='utf-8',header=None)
stop=pd.read_csv(stoplist,encoding='utf-8',header=None,sep='tipdm')
data_n[1]=data_n[0].apply(lambda s:s.split(' '))
data_n[2]=data_n[1].apply(lambda y: [i for i in y if i not in stop])
from gensim import corpora,models
dict_1=corpora.Dictionary(data_n[2])
dict_corpora=[dict_1.doc2bow(i) for i in data_n[2]]
data_lda=models.LdaModel(dict_corpora,num_topics=5,id2word=dict_1)
for j in range(5):
    print(data_lda.print_topic(j))


from collections import defaultdict
import os
import errno
import json
import time
import urllib.request
import urllib.error
import threading
from urllib.parse import urljoin

from ycmd.completers.completer import Completer
from ycmd.completers.completer_utils import GetFileLines
from ycmd.completers.cs import solutiondetection
from ycmd.utils import ( ByteOffsetToCodepointOffset,
                         CodepointOffsetToByteOffset,
                         FindExecutable,
                         FindExecutableWithFallback,
                         LOGGER,
                         ToBytes )
from ycmd import responses
from ycmd import utils

SERVER_NOT_FOUND_MSG = ( 'OmniSharp server binary not found at {0}. ' +
                         'Did you compile it? You can do so by running ' +
                         '"./install.py --cs-completer".' )
INVALID_FILE_MESSAGE = 'File is invalid.'
NO_DIAGNOSTIC_MESSAGE = 'No diagnostic for current line!'
PATH_TO_ROSLYN_OMNISHARP = os.path.join(
  os.path.abspath( os.path.dirname( __file__ ) ),
  '..', '..', '..', 'third_party', 'omnisharp-roslyn'
)
PATH_TO_OMNISHARP_ROSLYN_BINARY = os.path.join(
  PATH_TO_ROSLYN_OMNISHARP, 'Omnisharp.exe' )
if ( not os.path.isfile( PATH_TO_OMNISHARP_ROSLYN_BINARY )
     and os.path.isfile( os.path.join(
       PATH_TO_ROSLYN_OMNISHARP, 'omnisharp', 'OmniSharp.exe' ) ) ):
  PATH_TO_OMNISHARP_ROSLYN_BINARY = (
    os.path.join( PATH_TO_ROSLYN_OMNISHARP, 'omnisharp', 'OmniSharp.exe' ) )
LOGFILE_FORMAT = 'omnisharp_{port}_{sln}_{std}_'

def ShouldEnableCsCompleter( user_options ):
  user_roslyn_path = user_options[ 'roslyn_binary_path' ]
  if user_roslyn_path and not os.path.isfile( user_roslyn_path ):
    LOGGER.error( 'No omnisharp-roslyn executable at %s', user_roslyn_path )
    # We should trust the user who specifically asked for a custom path.
    return False

  if os.path.isfile( user_roslyn_path ):
    roslyn = user_roslyn_path
  else:
    roslyn = PATH_TO_OMNISHARP_ROSLYN_BINARY
  mono = FindExecutableWithFallback( user_options[ 'mono_binary_path' ],
                                     FindExecutable( 'mono' ) )
  if roslyn and ( mono or utils.OnWindows() ):
    return True
  LOGGER.info( 'No mono executable at %s', mono )
  return False

class CsharpCompleter( Completer ):
  """
  A Completer that uses the Omnisharp server as completion engine.
  """

  def __init__( self, user_options ):
    super().__init__( user_options )
    self._solution_for_file = {}
    self._completer_per_solution = {}
    self._diagnostic_store = None
    self._solution_state_lock = threading.Lock()
    self.SetSignatureHelpTriggers( [ '(', ',' ] )
    if os.path.isfile( user_options[ 'roslyn_binary_path' ] ):
      self._roslyn_path = user_options[ 'roslyn_binary_path' ]
    else:
      self._roslyn_path = PATH_TO_OMNISHARP_ROSLYN_BINARY
    self._mono_path = FindExecutableWithFallback(
        user_options[ 'mono_binary_path' ],
        FindExecutable( 'mono' ) )

  def Shutdown( self ):
    if self.user_options[ 'auto_stop_csharp_server' ]:
      for solutioncompleter in self._completer_per_solution.values():
        solutioncompleter._StopServer()

  def SupportedFiletypes( self ):
    """ Just csharp """
    return [ 'cs' ]

  def _GetSolutionCompleter( self, request_data ):
    """ Get the solution completer or create a new one if it does not already
    exist. Use a lock to avoid creating the same solution completer multiple
    times."""
    solution = self._GetSolutionFile( request_data[ "filepath" ] )

    with self._solution_state_lock:
      if solution not in self._completer_per_solution:
        keep_logfiles = self.user_options[ 'server_keep_logfiles' ]
        desired_omnisharp_port = self.user_options.get( 'csharp_server_port' )
        completer = CsharpSolutionCompleter( solution,
                                             keep_logfiles,
                                             desired_omnisharp_port,
                                             self._roslyn_path,
                                             self._mono_path )
        self._completer_per_solution[ solution ] = completer

    return self._completer_per_solution[ solution ]

  def SignatureHelpAvailable( self ):
    if not self.ServerIsHealthy():
      return responses.SignatureHelpAvailalability.PENDING
    return responses.SignatureHelpAvailalability.AVAILABLE

  def ComputeSignaturesInner( self, request_data ):
    response = self._SolutionSubcommand( request_data, '_SignatureHelp' )

    if response is None:
      return {}

    signatures = response[ 'Signatures' ]

    def MakeSignature( s ):
      sig_label = s[ 'Label' ]
      end = 0
      parameters = []
      for arg in s[ 'Parameters' ]:
        arg_label = arg[ 'Label' ]
        begin = sig_label.find( arg_label, end )
        end = begin + len( arg_label )
        parameters.append( {
          'documentation': arg.get( 'Documentation', '' ),
          'label': [ CodepointOffsetToByteOffset( sig_label, begin ),
                     CodepointOffsetToByteOffset( sig_label, end ) ]
        } )

      return {
        'documentation': s.get( 'Documentation', '' ),
        'label': sig_label,
        'parameters': parameters
      }

    return {
      'activeSignature': response[ 'ActiveSignature' ],
      'activeParameter': response[ 'ActiveParameter' ],
      'signatures': [ MakeSignature( s ) for s in signatures ]
    }

  def ResolveFixit( self, request_data ):
    return self._SolutionSubcommand( request_data, '_ResolveFixIt' )

  def ComputeCandidatesInner( self, request_data ):
    solutioncompleter = self._GetSolutionCompleter( request_data )
    return [ responses.BuildCompletionData(
                completion[ 'CompletionText' ],
                completion[ 'DisplayText' ],
                completion[ 'Description' ],
                None,
                completion[ 'Kind' ] )
             for completion
             in solutioncompleter._GetCompletions( request_data ) ]

  def GetSubcommandsMap( self ):
    return {
      'StopServer'                       : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_StopServer',
                                   no_request_data = True ) ),
      'RestartServer'                    : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_RestartServer',
                                   no_request_data = True ) ),
      'GoToDefinition'                   : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToDefinition' ) ),
      'GoToDeclaration'                  : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToDefinition' ) ),
      'GoTo'                             : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToImplementation',
                                   fallback_to_declaration = True ) ),
      'GoToDefinitionElseDeclaration'    : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToDefinition' ) ),
      'GoToReferences'                   : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToReferences' ) ),
      'GoToImplementation'               : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToImplementation',
                                   fallback_to_declaration = False ) ),
      'GoToImplementationElseDeclaration': ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToImplementation',
                                   fallback_to_declaration = True ) ),
      'GetType'                          : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GetType' ) ),
      'Format'                           : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_Format' ) ),
      'FixIt'                            : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_FixIt' ) ),
      'GetDoc'                           : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GetDoc' ) ),
      'GoToSymbol'                       : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_GoToSymbol',
                                   args = args ) ),
      'OrganizeImports'                  : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_OrganizeImports' ) ),
      'RefactorRename'                   : ( lambda self, request_data, args:
         self._SolutionSubcommand( request_data,
                                   method = '_RefactorRename',
                                   args = args ) ),
    }

  def _SolutionSubcommand( self, request_data, method,
                           no_request_data = False, **kwargs ):
    solutioncompleter = self._GetSolutionCompleter( request_data )
    if not no_request_data:
      kwargs[ 'request_data' ] = request_data
    return getattr( solutioncompleter, method )( **kwargs )

  def OnFileReadyToParse( self, request_data ):
    solutioncompleter = self._GetSolutionCompleter( request_data )

    # Only start the server associated to this solution if the option to
    # automatically start one is set and no server process is already running.
    if ( self.user_options[ 'auto_start_csharp_server' ]
         and not solutioncompleter._ServerIsRunning() ):
      solutioncompleter._StartServer()
      return

    # Bail out if the server is unresponsive. We don't start or restart the
    # server in this case because current one may still be warming up.
    if not solutioncompleter.ServerIsHealthy():
      return

    errors = solutioncompleter.CodeCheck( request_data )

    diagnostics = [ self._QuickFixToDiagnostic( request_data, x ) for x in
                    errors[ "QuickFixes" ] ]

    self._diagnostic_store = DiagnosticsToDiagStructure( diagnostics )

    return responses.BuildDiagnosticResponse( diagnostics,
                                              request_data[ 'filepath' ],
                                              self.max_diagnostics_to_display )

  def _QuickFixToDiagnostic( self, request_data, quick_fix ):
    filename = quick_fix[ "FileName" ]
    # NOTE: end of diagnostic range returned by the OmniSharp server is not
    # included.
    location = _BuildLocation( request_data,
                               filename,
                               quick_fix[ 'Line' ],
                               quick_fix[ 'Column' ] )
    location_end = _BuildLocation( request_data,
                                   filename,
                                   quick_fix[ 'EndLine' ],
                                   quick_fix[ 'EndColumn' ] )
    if not location_end:
      location_end = location
    location_extent = responses.Range( location, location_end )
    return responses.Diagnostic( [],
                                 location,
                                 location_extent,
                                 quick_fix[ 'Text' ],
                                 quick_fix[ 'LogLevel' ].upper() )

  def GetDetailedDiagnostic( self, request_data ):
    current_line = request_data[ 'line_num' ]
    current_column = request_data[ 'column_num' ]
    current_file = request_data[ 'filepath' ]

    if not self._diagnostic_store:
      raise ValueError( NO_DIAGNOSTIC_MESSAGE )

    diagnostics = self._diagnostic_store[ current_file ][ current_line ]
    if not diagnostics:
      raise ValueError( NO_DIAGNOSTIC_MESSAGE )

    closest_diagnostic = None
    distance_to_closest_diagnostic = 999

    # FIXME: all of these calculations are currently working with byte
    # offsets, which are technically incorrect. We should be working with
    # codepoint offsets, as we want the nearest character-wise diagnostic
    for diagnostic in diagnostics:
      distance = abs( current_column - diagnostic.location_.column_number_ )
      if distance < distance_to_closest_diagnostic:
        distance_to_closest_diagnostic = distance
        closest_diagnostic = diagnostic

    return responses.BuildDisplayMessageResponse(
      closest_diagnostic.text_ )

  def DebugInfo( self, request_data ):
    try:
      completer = self._GetSolutionCompleter( request_data )
    except RuntimeError:
      omnisharp_server = responses.DebugInfoServer(
        name = 'OmniSharp',
        handle = None,
        executable = self._roslyn_path )

      return responses.BuildDebugInfoResponse( name = 'C#',
                                               servers = [ omnisharp_server ] )

    with completer._server_state_lock:
      solution_item = responses.DebugInfoItem(
        key = 'solution',
        value = completer._solution_path )

      omnisharp_server = responses.DebugInfoServer(
        name = 'OmniSharp',
        handle = completer._omnisharp_phandle,
        executable = PATH_TO_ROSLYN_OMNISHARP,
        address = 'localhost',
        port = completer._omnisharp_port,
        logfiles = [ completer._filename_stdout, completer._filename_stderr ],
        extras = [ solution_item ] )

      return responses.BuildDebugInfoResponse( name = 'C#',
                                               servers = [ omnisharp_server ] )

  def ServerIsHealthy( self ):
    """ Check if our OmniSharp server is healthy (up and serving). """
    return self._CheckAllRunning( lambda i: i.ServerIsHealthy() )

  def ServerIsReady( self ):
    """ Check if our OmniSharp server is ready (loaded solution file)."""
    return self._CheckAllRunning( lambda i: i.ServerIsReady() )

  def _CheckAllRunning( self, action ):
    solutioncompleters = self._completer_per_solution.values()
    return all( action( completer ) for completer in solutioncompleters
                if completer._ServerIsRunning() )

  def _GetSolutionFile( self, filepath ):
    if filepath not in self._solution_for_file:
      # NOTE: detection could throw an exception if an extra_conf_store needs
      # to be confirmed
      path_to_solutionfile = solutiondetection.FindSolutionPath( filepath )
      if not path_to_solutionfile:
        raise RuntimeError( 'Autodetection of solution file failed.' )
      self._solution_for_file[ filepath ] = path_to_solutionfile

    return self._solution_for_file[ filepath ]

class CsharpSolutionCompleter( object ):
  def __init__( self,
                solution_path,
                keep_logfiles,
                desired_omnisharp_port,
                roslyn_path,
                mono_path ):
    self._solution_path = solution_path
    self._keep_logfiles = keep_logfiles
    self._filename_stderr = None
    self._filename_stdout = None
    self._omnisharp_port = None
    self._omnisharp_phandle = None
    self._desired_omnisharp_port = desired_omnisharp_port
    self._server_state_lock = threading.Lock()
    self._roslyn_path = roslyn_path
    self._mono_path = mono_path

  def CodeCheck( self, request_data ):
    filename = request_data[ 'filepath' ]
    if not filename:
      raise ValueError( INVALID_FILE_MESSAGE )

    return self._GetResponse( '/codecheck',
                              self._DefaultParameters( request_data ) )

  def _StartServer( self ):
    with self._server_state_lock:
      return self._StartServerNoLock()

  def _StartServerNoLock( self ):
    """ Start the OmniSharp server if not already running. Use a lock to avoid
    starting the server multiple times for the same solution. """
    if self._ServerIsRunning():
      return

    LOGGER.info( 'Starting OmniSharp server' )
    LOGGER.info( 'Loading solution file %s', self._solution_path )

    self._ChooseOmnisharpPort()

    command = [ PATH_TO_OMNISHARP_ROSLYN_BINARY,
                '-p',
                str( self._omnisharp_port ),
                '-s',
                str( self._solution_path ) ]

    if ( not utils.OnWindows()
         and self._roslyn_path.endswith( '.exe' ) ):
      command.insert( 0, self._mono_path )

    LOGGER.info( 'Starting OmniSharp server with: %s', command )

    solutionfile = os.path.basename( self._solution_path )
    self._filename_stdout = utils.CreateLogfile(
        LOGFILE_FORMAT.format( port = self._omnisharp_port,
                               sln = solutionfile,
                               std = 'stdout' ) )
    self._filename_stderr = utils.CreateLogfile(
        LOGFILE_FORMAT.format( port = self._omnisharp_port,
                               sln = solutionfile,
                               std = 'stderr' ) )

    with utils.OpenForStdHandle( self._filename_stderr ) as fstderr:
      with utils.OpenForStdHandle( self._filename_stdout ) as fstdout:
        self._omnisharp_phandle = utils.SafePopen(
            command, stdout = fstdout, stderr = fstderr )

    LOGGER.info( 'Started OmniSharp server' )

  def _StopServer( self ):
    with self._server_state_lock:
      return self._StopServerNoLock()

  def _StopServerNoLock( self ):
    """ Stop the OmniSharp server using a lock. """
    if self._ServerIsRunning():
      LOGGER.info( 'Stopping OmniSharp server with PID %s',
                   self._omnisharp_phandle.pid )
      try:
        self._TryToStopServer()
        self._ForceStopServer()
        utils.WaitUntilProcessIsTerminated( self._omnisharp_phandle,
                                            timeout = 5 )
        LOGGER.info( 'OmniSharp server stopped' )
      except Exception:
        LOGGER.exception( 'Error while stopping OmniSharp server' )

    self._CleanUp()

  def _TryToStopServer( self ):
    for _ in range( 5 ):
      try:
        self._GetResponse( '/stopserver', timeout = .5 )
      except Exception:
        pass
      for _ in range( 10 ):
        if not self._ServerIsRunning():
          return
        time.sleep( .1 )

  def _ForceStopServer( self ):
    # Kill it if it's still up
    phandle = self._omnisharp_phandle
    if phandle is not None:
      LOGGER.info( 'Killing OmniSharp server' )
      for stream in [ phandle.stderr, phandle.stdout ]:
        if stream is not None:
          stream.close()
      try:
        phandle.kill()
      except OSError as e:
        if e.errno == errno.ESRCH: # No such process
          pass
        else:
          raise

  def _CleanUp( self ):
    self._omnisharp_port = None
    self._omnisharp_phandle = None
    if not self._keep_logfiles:
      if self._filename_stdout:
        utils.RemoveIfExists( self._filename_stdout )
        self._filename_stdout = None
      if self._filename_stderr:
        utils.RemoveIfExists( self._filename_stderr )
        self._filename_stderr = None

  def _RestartServer( self ):
    """ Restarts the OmniSharp server using a lock. """
    with self._server_state_lock:
      self._StopServerNoLock()
      return self._StartServerNoLock()

  def _GetCompletions( self, request_data ):
    """ Ask server for completions """
    parameters = self._DefaultParameters( request_data )
    parameters[ 'WantSnippet' ] = False
    parameters[ 'WantKind' ] = True
    parameters[ 'WantReturnType' ] = False
    parameters[ 'WantDocumentationForEveryCompletionResult' ] = True
    completions = self._GetResponse( '/autocomplete', parameters )
    return completions if completions is not None else []

  def _GoToDefinition( self, request_data ):
    """ Jump to definition of identifier under cursor """
    definition = self._GetResponse( '/gotodefinition',
                                    self._DefaultParameters( request_data ) )
    if definition[ 'FileName' ] is not None:
      filepath = definition[ 'FileName' ]
      return responses.BuildGoToResponseFromLocation(
        _BuildLocation( request_data,
                        filepath,
                        definition[ 'Line' ],
                        definition[ 'Column' ] ) )
    else:
      raise RuntimeError( 'Can\'t jump to definition' )

  def _GoToImplementation( self, request_data, fallback_to_declaration ):
    """ Jump to implementation of identifier under cursor """
    try:
      implementation = self._GetResponse(
          '/findimplementations',
          self._DefaultParameters( request_data ) )
    except ValueError:
      implementation = { 'QuickFixes': None }

    quickfixes = implementation[ 'QuickFixes' ]
    if quickfixes:
      if len( quickfixes ) == 1:
        impl = quickfixes[ 0 ]
        return responses.BuildGoToResponseFromLocation(
          _BuildLocation(
            request_data,
            impl[ 'FileName' ],
            impl[ 'Line' ],
            impl[ 'Column' ] ) )
      else:
        return [ responses.BuildGoToResponseFromLocation(
                   _BuildLocation( request_data,
                                   x[ 'FileName' ],
                                   x[ 'Line' ],
                                   x[ 'Column' ] ) )
                 for x in quickfixes ]
    else:
      if ( fallback_to_declaration ):
        return self._GoToDefinition( request_data )
      elif quickfixes is None:
        raise RuntimeError( 'Can\'t jump to implementation' )
      else:
        raise RuntimeError( 'No implementations found' )

  def _SignatureHelp( self, request_data ):
    request = self._DefaultParameters( request_data )
    return self._GetResponse( '/signatureHelp', request )

  def _RefactorRename( self, request_data, args ):
    request = self._DefaultParameters( request_data )
    if len( args ) != 1:
      raise ValueError( 'Please specify a new name to rename it to.\n'
                        'Usage: RefactorRename <new name>' )
    request[ 'RenameTo' ] = args[ 0 ]
    request[ 'WantsTextChanges' ] = True
    response = self._GetResponse( '/rename', request )
    fixit = _ModifiedFilesToFixIt( response[ 'Changes' ], request_data )
    return responses.BuildFixItResponse( [ fixit ] )

  def _GoToSymbol( self, request_data, args ):
    request = self._DefaultParameters( request_data )
    request.update( {
      'Language': 'C#',
      'Filter': args[ 0 ]
    } )
    response = self._GetResponse( '/findsymbols', request )

    quickfixes = response[ 'QuickFixes' ]
    if quickfixes:
      if len( quickfixes ) == 1:
        ref = quickfixes[ 0 ]
        ref_file = ref[ 'FileName' ]
        ref_line = ref[ 'Line' ]
        lines = GetFileLines( request_data, ref_file )
        line = lines[ min( len( lines ), ref_line - 1 ) ]
        return responses.BuildGoToResponseFromLocation(
          _BuildLocation(
            request_data,
            ref_file,
            ref_line,
            ref[ 'Column' ] ),
          line )
      else:
        goto_locations = []
        for ref in quickfixes:
          ref_file = ref[ 'FileName' ]
          ref_line = ref[ 'Line' ]
          lines = GetFileLines( request_data, ref_file )
          line = lines[ min( len( lines ), ref_line - 1 ) ]
          goto_locations.append(
            responses.BuildGoToResponseFromLocation(
              _BuildLocation( request_data,
                              ref_file,
                              ref_line,
                              ref[ 'Column' ] ),
              line ) )

        return goto_locations
    else:
      raise RuntimeError( 'No symbols found' )

  def _GoToReferences( self, request_data ):
    """ Jump to references of identifier under cursor """
    # _GetResponse can throw. Original code by @mispencer
    # wrapped it in a try/except and set `reference` to `{ 'QuickFixes': None }`
    # After being unable to hit that case with tests,
    # that code path was thrown away.
    reference = self._GetResponse(
       '/findusages',
       self._DefaultParameters( request_data ) )

    quickfixes = reference[ 'QuickFixes' ]
    if quickfixes:
      if len( quickfixes ) == 1:
        ref = quickfixes[ 0 ]
        return responses.BuildGoToResponseFromLocation(
          _BuildLocation(
            request_data,
            ref[ 'FileName' ],
            ref[ 'Line' ],
            ref[ 'Column' ] ) )
      else:
        return [ responses.BuildGoToResponseFromLocation(
                   _BuildLocation( request_data,
                                   ref[ 'FileName' ],
                                   ref[ 'Line' ],
                                   ref[ 'Column' ] ) )
                 for ref in quickfixes ]
    else:
      raise RuntimeError( 'No references found' )

  def _GetType( self, request_data ):
    request = self._DefaultParameters( request_data )
    request[ "IncludeDocumentation" ] = False

    result = self._GetResponse( '/typelookup', request )
    message = result[ "Type" ]

    if not message:
      raise RuntimeError( 'No type info available.' )
    return responses.BuildDisplayMessageResponse( message )

  def _OrganizeImports( self, request_data ):
    request = self._DefaultParameters( request_data )
    request[ 'WantsTextChanges' ] = True
    result = self._GetResponse( '/fixusings', request )
    fixit = responses.FixIt(
      _BuildLocation(
        request_data,
        request_data[ 'filepath' ],
        request_data[ 'line_num' ],
        request_data[ 'column_codepoint' ] ),
      _LinePositionSpanTextChangeToFixItChunks(
        result[ 'Changes' ],
        request_data[ 'filepath' ],
        request_data ) )
    return responses.BuildFixItResponse( [ fixit ] )

  def _Format( self, request_data ):
    request = self._DefaultParameters( request_data )
    request[ 'WantsTextChanges' ] = True
    if 'range' in request_data:
      lines = request_data[ 'lines' ]
      start = request_data[ 'range' ][ 'start' ]
      start_line_num = start[ 'line_num' ]
      start_line_value = lines[ start_line_num ]

      start_codepoint = ByteOffsetToCodepointOffset( start_line_value,
                                                     start[ 'column_num' ] )

      end = request_data[ 'range' ][ 'end' ]
      end_line_num = end[ 'line_num' ]
      end_line_value = lines[ end_line_num ]
      end_codepoint = ByteOffsetToCodepointOffset( end_line_value,
                                                   end[ 'column_num' ] )
      request.update( {
        'line': start_line_num,
        'column': start_codepoint,
        'EndLine': end_line_num,
        'EndColumn': end_codepoint
      } )
      result = self._GetResponse( '/formatRange', request )
    else:
      result = self._GetResponse( '/codeformat', request )

    fixit = responses.FixIt(
      _BuildLocation(
        request_data,
        request_data[ 'filepath' ],
        request_data[ 'line_num' ],
        request_data[ 'column_codepoint' ] ),
      _LinePositionSpanTextChangeToFixItChunks(
        result[ 'Changes' ],
        request_data[ 'filepath' ],
        request_data ) )
    return responses.BuildFixItResponse( [ fixit ] )

  def _FixIt( self, request_data ):
    request = self._DefaultParameters( request_data )
    request[ 'WantsTextChanges' ] = True

    result = self._GetResponse( '/getcodeactions', request )

    fixits = []
    for i, code_action_name in enumerate( result[ 'CodeActions' ] ):
      fixit = responses.UnresolvedFixIt( { 'index': i }, code_action_name )
      fixits.append( fixit )

    if len( fixits ) == 1:
      fixit = fixits[ 0 ]
      fixit = { 'command': fixit.command, 'resolve': fixit.resolve }
      return self._ResolveFixIt( request_data, fixit )

    return responses.BuildFixItResponse( fixits )

  def _ResolveFixIt( self, request_data, unresolved_fixit = None ):
    fixit = unresolved_fixit if unresolved_fixit else request_data[ 'fixit' ]
    if not fixit[ 'resolve' ]:
      return { 'fixits': [ fixit ] }
    fixit = fixit[ 'command' ]
    code_action = fixit[ 'index' ]
    request = self._DefaultParameters( request_data )
    request.update( {
      'CodeAction': code_action,
      'WantsTextChanges': True,
    } )
    response = self._GetResponse( '/runcodeaction', request )
    fixit = responses.FixIt(
      _BuildLocation(
        request_data,
        request_data[ 'filepath' ],
        request_data[ 'line_num' ],
        request_data[ 'column_codepoint' ] ),
      _LinePositionSpanTextChangeToFixItChunks(
        response[ 'Changes' ],
        request_data[ 'filepath' ],
        request_data ),
      response[ 'Text' ] )
    # The sort is necessary to keep the tests stable.
    # Python's sort() is stable, so it won't mess up the order within a file.
    fixit.chunks.sort( key = lambda c: c.range.start_.filename_ )
    return responses.BuildFixItResponse( [ fixit ] )

  def _GetDoc( self, request_data ):
    request = self._DefaultParameters( request_data )
    request[ "IncludeDocumentation" ] = True

    result = self._GetResponse( '/typelookup', request )
    message = result.get( 'Type' ) or ''

    if ( result[ "Documentation" ] ):
      message += "\n" + result[ "Documentation" ]

    if not message:
      raise RuntimeError( 'No documentation available.' )
    return responses.BuildDetailedInfoResponse( message.strip() )

  def _DefaultParameters( self, request_data ):
    """ Some very common request parameters """
    parameters = {}
    parameters[ 'line' ] = request_data[ 'line_num' ]
    parameters[ 'column' ] = request_data[ 'column_codepoint' ]

    filepath = request_data[ 'filepath' ]
    parameters[ 'buffer' ] = (
      request_data[ 'file_data' ][ filepath ][ 'contents' ] )
    parameters[ 'filename' ] = filepath
    return parameters

  def _ServerIsRunning( self ):
    """ Check if our OmniSharp server is running (process is up)."""
    return utils.ProcessIsRunning( self._omnisharp_phandle )

  def ServerIsHealthy( self ):
    """ Check if our OmniSharp server is healthy (up and serving)."""
    if not self._ServerIsRunning():
      return False

    try:
      return self._GetResponse( '/checkalivestatus', timeout = 3 )
    except Exception:
      return False

  def ServerIsReady( self ):
    """ Check if our OmniSharp server is ready (loaded solution file)."""
    if not self._ServerIsRunning():
      return False

    try:
      return self._GetResponse( '/checkreadystatus', timeout = .2 )
    except Exception:
      return False

  def _ServerLocation( self ):
    # We cannot use 127.0.0.1 like we do in other places because OmniSharp
    # server only listens on localhost.
    return f'http://127.0.0.1:{ self._omnisharp_port }'

  def _GetResponse( self, handler, parameters = {}, timeout = None ):
    """ Handle communication with server """
    target = urljoin( self._ServerLocation(), handler )
    LOGGER.debug( 'TX (%s): %s', handler, parameters )
    try:
      response = urllib.request.urlopen(
        target,
        data = ToBytes( json.dumps( parameters ) ),
        timeout = timeout )
      json_response = json.loads( response.read() )
      response.close()
    except urllib.error.HTTPError as response:
      json_response = json.loads( response.fp.read() )
      response.close()
    LOGGER.debug( 'RX: %s', json_response )
    return json_response

  def _ChooseOmnisharpPort( self ):
    if not self._omnisharp_port:
      if self._desired_omnisharp_port:
        self._omnisharp_port = int( self._desired_omnisharp_port )
      else:
        self._omnisharp_port = utils.GetUnusedLocalhostPort()
    LOGGER.info( 'using port %s', self._omnisharp_port )

def DiagnosticsToDiagStructure( diagnostics ):
  structure = defaultdict( lambda : defaultdict( list ) )
  for diagnostic in diagnostics:
    structure[ diagnostic.location_.filename_ ][
      diagnostic.location_.line_number_ ].append( diagnostic )
  return structure

def _BuildLocation( request_data, filename, line_num, column_num ):
  if line_num <= 0:
    return None
  # OmniSharp sometimes incorrectly returns 0 for the column number. Assume the
  # column is 1 in that case.
  if column_num <= 0:
    column_num = 1
  contents = GetFileLines( request_data, filename )
  line_value = contents[ min( len( contents ) - 1, line_num - 1 ) ]
  return responses.Location(
      line_num,
      CodepointOffsetToByteOffset( line_value, column_num ),
      filename )

def _LinePositionSpanTextChangeToFixItChunks( chunks, filename, request_data ):
  return [ responses.FixItChunk(
      chunk[ 'NewText' ],
      responses.Range(
        _BuildLocation(
          request_data,
          filename,
          chunk[ 'StartLine' ],
          chunk[ 'StartColumn' ] ),
        _BuildLocation(
          request_data,
          filename,
          chunk[ 'EndLine' ],
          chunk[ 'EndColumn' ] ) ) ) for chunk in chunks ]

def _ModifiedFilesToFixIt( changes, request_data ):
  chunks = []
  for change in changes:
    chunks.extend(
      _LinePositionSpanTextChangeToFixItChunks(
        change[ 'Changes' ],
        change[ 'FileName' ],
        request_data ) )
  # The sort is necessary to keep the tests stable.
  # Python's sort() is stable, so it won't mess up the order within a file.
  chunks.sort( key = lambda c: c.range.start_.filename_ )
  return responses.FixIt(
      _BuildLocation(
        request_data,
        request_data[ 'filepath' ],
        request_data[ 'line_num' ],
        request_data[ 'column_codepoint' ] ),
      chunks )


from typing import Dict
import logging
import numpy as np

from src.utils import flat_array
from src.sinks.sink_slices import SinkSlices

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()

class SinkDictFlat(SinkSlices):
    """
        A sink filter that saves a list of dictionnaries into a flat npy file
    """

    def __init__(self, conf_filter: Dict = {}):
        self.l_keys = conf_filter['l_keys']
        super().__init__(conf_filter)

    def process(self, message: Dict) -> Dict:
        dict_to_save = {}
        for key in self.l_keys:
            dict_to_save[key] = message[key]
        self.collect_data.append(dict_to_save)

        if len(self.collect_data)>=self.slice_length:
            self._save_slice()
        return message