import os
import cv2
import colorsys
import numpy as np
import time
import paddlex.utils.logging as logging
from .detection_eval import fixed_linspace, backup_linspace, loadRes
from paddlex.cv.datasets.dataset import is_pic

def visualize_detection(image,
                        result,
                        threshold=0.5,
                        save_dir='./',
                        color=None):
    """
        Visualize bbox and mask results
    """

    if isinstance(image, np.ndarray):
        image_name = str(int(time.time() * 1000)) + '.jpg'
    else:
        image_name = os.path.split(image)[-1]
        image = cv2.imread(image)

    image = draw_bbox_mask(image, result, threshold=threshold, color_map=color)
    if save_dir is not None:
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
        out_path = os.path.join(save_dir, 'visualize_{}'.format(image_name))
        cv2.imwrite(out_path, image)
        logging.info('The visualized result is saved as {}'.format(out_path))
    else:
        return image

def visualize_segmentation(image,
                           result,
                           weight=0.6,
                           save_dir='./',
                           color=None):
    """
    Convert segment result to color image, and save added image.
    Args:
        image: the path of origin image
        result: the predict result of image
        weight: the image weight of visual image, and the result weight is (1 - weight)
        save_dir: the directory for saving visual image
        color: the list of a BGR-mode color for each label.
    """
    label_map = result['label_map']
    color_map = get_color_map_list(256)
    if color is not None:
        for i in range(len(color) // 3):
            color_map[i] = color[i * 3:(i + 1) * 3]
    color_map = np.array(color_map).astype("uint8")

    # Use OpenCV LUT for color mapping
    c1 = cv2.LUT(label_map, color_map[:, 0])
    c2 = cv2.LUT(label_map, color_map[:, 1])
    c3 = cv2.LUT(label_map, color_map[:, 2])
    pseudo_img = np.dstack((c1, c2, c3))

    if isinstance(image, np.ndarray):
        im = image
        image_name = str(int(time.time() * 1000)) + '.jpg'
        if image.shape[2] != 3:
            logging.info(
                "The image is not 3-channel array, so predicted label map is shown as a pseudo color image."
            )
            weight = 0.
    else:
        image_name = os.path.split(image)[-1]
        if not is_pic(image):
            logging.info(
                "The image cannot be opened by opencv, so predicted label map is shown as a pseudo color image."
            )
            image_name = image_name.split('.')[0] + '.jpg'
            weight = 0.
        else:
            im = cv2.imread(image)

    if abs(weight) < 1e-5:
        vis_result = pseudo_img
    else:
        vis_result = cv2.addWeighted(im, weight,
                                     pseudo_img.astype(im.dtype), 1 - weight,
                                     0)

    if save_dir is not None:
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
        out_path = os.path.join(save_dir, 'visualize_{}'.format(image_name))
        cv2.imwrite(out_path, vis_result)
        logging.info('The visualized result is saved as {}'.format(out_path))
    else:
        return vis_result

def get_color_map_list(num_classes):
    """ Returns the color map for visualizing the segmentation mask,
        which can support arbitrary number of classes.
    Args:
        num_classes: Number of classes
    Returns:
        The color map
    """
    color_map = num_classes * [0, 0, 0]
    for i in range(0, num_classes):
        j = 0
        lab = i
        while lab:
            color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j))
            color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j))
            color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j))
            j += 1
            lab >>= 3
    color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
    return color_map

# expand an array of boxes by a given scale.
def expand_boxes(boxes, scale):
    """
        """
    w_half = (boxes[:, 2] - boxes[:, 0]) * .5
    h_half = (boxes[:, 3] - boxes[:, 1]) * .5
    x_c = (boxes[:, 2] + boxes[:, 0]) * .5
    y_c = (boxes[:, 3] + boxes[:, 1]) * .5

    w_half *= scale
    h_half *= scale

    boxes_exp = np.zeros(boxes.shape)
    boxes_exp[:, 0] = x_c - w_half
    boxes_exp[:, 2] = x_c + w_half
    boxes_exp[:, 1] = y_c - h_half
    boxes_exp[:, 3] = y_c + h_half

    return boxes_exp

def clip_bbox(bbox):
    xmin = max(min(bbox[0], 1.), 0.)
    ymin = max(min(bbox[1], 1.), 0.)
    xmax = max(min(bbox[2], 1.), 0.)
    ymax = max(min(bbox[3], 1.), 0.)
    return xmin, ymin, xmax, ymax

def draw_bbox_mask(image, results, threshold=0.5, color_map=None):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib as mpl
    import matplotlib.figure as mplfigure
    import matplotlib.colors as mplc
    from matplotlib.backends.backend_agg import FigureCanvasAgg

    # refer to  https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/visualizer.py
    def _change_color_brightness(color, brightness_factor):
        assert brightness_factor >= -1.0 and brightness_factor <= 1.0
        color = mplc.to_rgb(color)
        polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color))
        modified_lightness = polygon_color[1] + (brightness_factor *
                                                 polygon_color[1])
        modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness
        modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness
        modified_color = colorsys.hls_to_rgb(
            polygon_color[0], modified_lightness, polygon_color[2])
        return modified_color

    _SMALL_OBJECT_AREA_THRESH = 1000
    # setup figure
    width, height = image.shape[1], image.shape[0]
    scale = 1
    fig = mplfigure.Figure(frameon=False)
    dpi = fig.get_dpi()
    fig.set_size_inches(
        (width * scale + 1e-2) / dpi,
        (height * scale + 1e-2) / dpi, )
    canvas = FigureCanvasAgg(fig)
    ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
    ax.axis("off")
    ax.set_xlim(0.0, width)
    ax.set_ylim(height)
    default_font_size = max(np.sqrt(height * width) // 90, 10 // scale)
    linewidth = max(default_font_size / 4, 1)

    labels = list()
    for dt in np.array(results):
        if dt['category'] not in labels:
            labels.append(dt['category'])

    if color_map is None:
        color_map = get_color_map_list(len(labels) + 2)[2:]
    else:
        color_map = np.asarray(color_map)
        if color_map.shape[0] != len(labels) or color_map.shape[1] != 3:
            raise Exception(
                "The shape for color_map is required to be {}x3, but recieved shape is {}x{}.".
                format(len(labels), color_map.shape))
        if np.max(color_map) > 255 or np.min(color_map) < 0:
            raise ValueError(
                " The values in color_map should be within 0-255 range.")

    keep_results = []
    areas = []
    for dt in np.array(results):
        cname, bbox, score = dt['category'], dt['bbox'], dt['score']
        if score < threshold:
            continue
        keep_results.append(dt)
        areas.append(bbox[2] * bbox[3])
    areas = np.asarray(areas)
    sorted_idxs = np.argsort(-areas).tolist()
    keep_results = [keep_results[k]
                    for k in sorted_idxs] if len(keep_results) > 0 else []

    for dt in np.array(keep_results):
        cname, bbox, score = dt['category'], dt['bbox'], dt['score']
        xmin, ymin, w, h = bbox
        xmax = xmin + w
        ymax = ymin + h

        color = tuple(color_map[labels.index(cname)])
        color = [c / 255. for c in color]
        # draw bbox
        ax.add_patch(
            mpl.patches.Rectangle(
                (xmin, ymin),
                w,
                h,
                fill=False,
                edgecolor=color,
                linewidth=linewidth * scale,
                alpha=0.8,
                linestyle="-", ))

        # draw mask
        if 'mask' in dt:
            mask = dt['mask']
            mask = np.ascontiguousarray(mask)
            res = cv2.findContours(
                mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)
            hierarchy = res[-1]
            alpha = 0.5
            if hierarchy is not None:
                has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0
                res = res[-2]
                res = [x.flatten() for x in res]
                res = [x for x in res if len(x) >= 6]
                for segment in res:
                    segment = segment.reshape(-1, 2)
                    edge_color = mplc.to_rgb(color) + (1, )
                    polygon = mpl.patches.Polygon(
                        segment,
                        fill=True,
                        facecolor=mplc.to_rgb(color) + (alpha, ),
                        edgecolor=edge_color,
                        linewidth=max(default_font_size // 15 * scale, 1), )
                    ax.add_patch(polygon)

        # draw label
        text_pos = (xmin, ymin)
        horiz_align = "left"
        instance_area = w * h
        if (instance_area < _SMALL_OBJECT_AREA_THRESH * scale or
                h < 40 * scale):
            if ymin >= height - 5:
                text_pos = (xmin, ymin)
            else:
                text_pos = (xmin, ymax)
        height_ratio = h / np.sqrt(height * width)
        font_size = (np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2,
                             2) * 0.5 * default_font_size)
        text = "{} {:.2f}".format(cname, score)
        color = np.maximum(list(mplc.to_rgb(color)), 0.2)
        color[np.argmax(color)] = max(0.8, np.max(color))
        color = _change_color_brightness(color, brightness_factor=0.7)
        ax.text(
            text_pos[0],
            text_pos[1],
            text,
            size=font_size * scale,
            family="sans-serif",
            bbox={
                "facecolor": "black",
                "alpha": 0.8,
                "pad": 0.7,
                "edgecolor": "none"
            },
            verticalalignment="top",
            horizontalalignment=horiz_align,
            color=color,
            zorder=10,
            rotation=0, )

    s, (width, height) = canvas.print_to_buffer()
    buffer = np.frombuffer(s, dtype="uint8")

    img_rgba = buffer.reshape(height, width, 4)
    rgb, alpha = np.split(img_rgba, [3], axis=2)

    try:
        import numexpr as ne
        visualized_image = ne.evaluate(
            "image * (1 - alpha / 255.0) + rgb * (alpha / 255.0)")
    except ImportError:
        alpha = alpha.astype("float32") / 255.0
        visualized_image = image * (1 - alpha) + rgb * alpha

    visualized_image = visualized_image.astype("uint8")

    return visualized_image

def draw_pr_curve(eval_details_file=None,
                  gt=None,
                  pred_bbox=None,
                  pred_mask=None,
                  iou_thresh=0.5,
                  save_dir='./'):
    if eval_details_file is not None:
        import json
        with open(eval_details_file, 'r') as f:
            eval_details = json.load(f)
            pred_bbox = eval_details['bbox']
            if 'mask' in eval_details:
                pred_mask = eval_details['mask']
            gt = eval_details['gt']
    if gt is None or pred_bbox is None:
        raise Exception(
            "gt/pred_bbox/pred_mask is None now, please set right eval_details_file or gt/pred_bbox/pred_mask."
        )
    if pred_bbox is not None and len(pred_bbox) == 0:
        raise Exception("There is no predicted bbox.")
    if pred_mask is not None and len(pred_mask) == 0:
        raise Exception("There is no predicted mask.")
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval
    coco = COCO()
    coco.dataset = gt
    coco.createIndex()

    def _summarize(coco_gt, ap=1, iouThr=None, areaRng='all', maxDets=100):
        p = coco_gt.params
        aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
        mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
        if ap == 1:
            # dimension of precision: [TxRxKxAxM]
            s = coco_gt.eval['precision']
            # IoU
            if iouThr is not None:
                t = np.where(iouThr == p.iouThrs)[0]
                s = s[t]
            s = s[:, :, :, aind, mind]
        else:
            # dimension of recall: [TxKxAxM]
            s = coco_gt.eval['recall']
            if iouThr is not None:
                t = np.where(iouThr == p.iouThrs)[0]
                s = s[t]
            s = s[:, :, aind, mind]
        if len(s[s > -1]) == 0:
            mean_s = -1
        else:
            mean_s = np.mean(s[s > -1])
        return mean_s

    def cal_pr(coco_gt, coco_dt, iou_thresh, save_dir, style='bbox'):
        coco_dt = loadRes(coco_gt, coco_dt)
        np.linspace = fixed_linspace
        coco_eval = COCOeval(coco_gt, coco_dt, style)
        coco_eval.params.iouThrs = np.linspace(
            iou_thresh, iou_thresh, 1, endpoint=True)
        np.linspace = backup_linspace
        coco_eval.evaluate()
        coco_eval.accumulate()
        stats = _summarize(coco_eval, iouThr=iou_thresh)
        catIds = coco_gt.getCatIds()
        if len(catIds) != coco_eval.eval['precision'].shape[2]:
            raise Exception(
                "The category number must be same as the third dimension of precisions."
            )
        x = np.arange(0.0, 1.01, 0.01)
        color_map = get_color_map_list(256)[1:256]

        plt.subplot(1, 2, 1)
        plt.title(style + " precision-recall IoU={}".format(iou_thresh))
        plt.xlabel("recall")
        plt.ylabel("precision")
        plt.xlim(0, 1.01)
        plt.ylim(0, 1.01)
        plt.grid(linestyle='--', linewidth=1)
        plt.plot([0, 1], [0, 1], 'r--', linewidth=1)
        my_x_ticks = np.arange(0, 1.01, 0.1)
        my_y_ticks = np.arange(0, 1.01, 0.1)
        plt.xticks(my_x_ticks, fontsize=5)
        plt.yticks(my_y_ticks, fontsize=5)
        for idx, catId in enumerate(catIds):
            pr_array = coco_eval.eval['precision'][0, :, idx, 0, 2]
            precision = pr_array[pr_array > -1]
            ap = np.mean(precision) if precision.size else float('nan')
            nm = coco_gt.loadCats(catId)[0]['name'] + ' AP={:0.2f}'.format(
                float(ap * 100))
            color = tuple(color_map[idx])
            color = [float(c) / 255 for c in color]
            color.append(0.75)
            plt.plot(x, pr_array, color=color, label=nm, linewidth=1)
        plt.legend(loc="lower left", fontsize=5)

        plt.subplot(1, 2, 2)
        plt.title(style + " score-recall IoU={}".format(iou_thresh))
        plt.xlabel('recall')
        plt.ylabel('score')
        plt.xlim(0, 1.01)
        plt.ylim(0, 1.01)
        plt.grid(linestyle='--', linewidth=1)
        plt.xticks(my_x_ticks, fontsize=5)
        plt.yticks(my_y_ticks, fontsize=5)
        for idx, catId in enumerate(catIds):
            nm = coco_gt.loadCats(catId)[0]['name']
            sr_array = coco_eval.eval['scores'][0, :, idx, 0, 2]
            color = tuple(color_map[idx])
            color = [float(c) / 255 for c in color]
            color.append(0.75)
            plt.plot(x, sr_array, color=color, label=nm, linewidth=1)
        plt.legend(loc="lower left", fontsize=5)
        plt.savefig(
            os.path.join(
                save_dir,
                "./{}_pr_curve(iou-{}).png".format(style, iou_thresh)),
            dpi=800)
        plt.close()

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    cal_pr(coco, pred_bbox, iou_thresh, save_dir, style='bbox')
    if pred_mask is not None:
        cal_pr(coco, pred_mask, iou_thresh, save_dir, style='segm')


# -*- coding: utf-8 -*-
#imports - consolidation of joint work by Steve and Avi
import pandas as pd
import numpy as np
import string

#read_file is a helper function to get the '|' delimited CSV into a data frame
def read_file(filename):
    #get the file
    df = pd.read_csv(filename, low_memory=False, delimiter = ',')
    #filter the null data
    filtered_data = df["RawText"].notnull()
    df_narrative = df[filtered_data]
    return df_narrative

#reads in the training data into a panda - Steve
#(code based on ANLP Notebook Intro to Pandas by Marti Hearst and Andrea Gagliano)
def read_training_data(filename):
    df_narrative = read_file(filename)
    #print the report on category breakdown, might need these counts later
#    print("Creating training data... category breakdown:")
#    sorted_product_counts = df_narrative.Category.value_counts(ascending=True)
#    print(sorted_product_counts)
    #sorted_product_counts.plot(kind='barh', figsize=(8,6), title="Categories");
    return df_narrative

#reads in the test file into a panda - Steve (code based on ANLP Notebook Intro to Pandas by Marti Hearst and Andrea Gagliano)
def read_test_data(filename):
    #get the file
    df = read_file(filename)
    return df

def get_number(prompt):
    user = input(prompt)                   #get input from keyboard
    answers = {'1','2','3'}
    while user not in answers:    #make sure something was entered
        user = input(prompt)   #else, prompt again
    return user

#breaks the panda into a training set and a dev set -Steve
#(code based on ANLP Notebook Intro to Pandas by Marti Hearst and Andrea Gagliano)
def get_labelling_sets(full_data, percent):
    #randomize the indices
    #break down the counts for the shuffled data
    rows, columns = full_data.shape
    train_size = round(rows*(1 - percent))
    #separate the training data from the development data
    train_data = full_data.loc[train_size:]
    print("Loading emails "+str(train_size)+" to "+str(train_size*2))
    rows, colums = train_data.shape
    train_size = round(rows*.33)
    print("Dividing emails into batches of "+str(train_size))
    return train_data[:train_size], train_data[train_size:train_size*2], train_data[train_size*2:train_size*3]

def get_random_emails(data_set, number):
    random_index = np.random.permutation(data_set.index)
    full_data_shuffled = data_set.ix[random_index,\
    ['Id', 'DocNumber', 'MetadataSubject', 'MetadataTo', 'Metadata From', 'MetadataDateSent', 'ExtractedSubject', 'ExtractedTo', 'ExtractedFrom', 'ExtractedBodyText','RawText']]
    full_data_shuffled.reset_index(drop=True, inplace=True)
    print("Giving you "+str(number)+" of emails to label...")
    #separate the training data from the development data
    return full_data_shuffled.loc[0:number-1]

def get_category(prompt):
    user = input(prompt)                   #get input from keyboard
    good = {'0','1'}
    answers = {''}
    chk_prompt = "Are you sure? (Hit Enter again to confirm -or- N for No)"
    confirm = input(chk_prompt)
    while confirm not in answers or user not in good:    #make sure something was entered
        user = input(prompt)   #else, prompt again
        confirm = input(chk_prompt)
    return user

def label_categories(data_set, number):
    cats = ['0. neutral', '1. AWESOMENESS']

    results = []
    data_list = data_set["RawText"].values.tolist()
    for index in range(0, number):
        print(cats)
        printable = set(string.printable)
        s = "".join(filter(lambda x: x in printable, data_list[index]))
        print(s)
        mycat = get_category('Email #'+str(index+1)+' Enter category (an integer, please): ')
        print('You labeled it ' + cats[int(mycat)])
        print('\n')
        results.append(int(mycat))

    print(results)
    return results

#the code that calls the above functions - puts the data into a data frame
df = read_training_data("Emails.csv")
shrestha_set, shannon_set, steve_set = get_labelling_sets(df,.5)
user = get_number("Enter 1 for Shrestha, 2 for Shannon, 3 for Steve: ")
if user == '1':
    my_set = shrestha_set
if user == '2':
    my_set = shannon_set
if user == '3':
    my_set = steve_set

num = 233
selected_emails = get_random_emails(my_set, num)
print(selected_emails.shape)
results = label_categories(selected_emails, num)

selected_emails['Label'] = ""
print(selected_emails['Label'])
selected_emails['Label'] = results
print(selected_emails.head())
selected_emails.to_csv('email_out'+user+'set2.csv' , index = False)


#!/bin/python3

# Complete the rotLeft function below.
def rotLeft(a, d):
    for i in range(d):
        value = a[0]
        a.pop(0)
        a.insert(len(a),value)
    return a

nd = input().split()
n = int(nd[0])
d = int(nd[1])
a = list(map(int, input().rstrip().split()))

result = rotLeft(a, d)
print(' '.join(map(str, result)))


from .JSONResponse import JSONResponse
from match.serializers import UserSerializer,UserProfileSerializer

from django.conf.urls import include,url
from django.contrib.auth.models import Group,User
from oauth2_provider.ext.rest_framework import TokenHasReadWriteScope, TokenHasScope
from rest_framework import decorators,permissions,routers,status,viewsets
from rest_framework.routers import DefaultRouter

class UserViewSet(viewsets.ModelViewSet):
    permission_classes = [permissions.IsAuthenticated, TokenHasScope]
    required_scopes = ['admin']
    queryset = User.objects.select_related('profile').all()

    serializer_class = UserSerializer

    def list(self, request):
        if request.user and request.user.is_staff:
            return super(viewsets.ModelViewSet, self).list(request)
        else:
            return JSONResponse({"detail": "You need to be an admin to view this content."}, status=status.HTTP_403_FORBIDDEN)

    def create(self, request):
            userdata = dict(request.data)
            userdata.pop("profile", None)
            serializer = UserSerializer(data=userdata)
            serializer.is_valid(raise_exception=True)
            serializer.save()
            if 'profile' in serializer.validated_data:
                up_serializer = UserProfileSerializer(data=request.user.profile)
                up_serializer.is_valid(raise_exception=True)
                profile = User.objects.get(username=request.data.email)
                up_serializer.update(profile, request.user.profile)
                serializer = UserSerializer(User.objects.get(username=serializer.email))
                return JSONResponse(serializer.data, status=status.HTTP_201_CREATED)

            else:
                return JSONResponse(serializer.data, status=status.HTTP_201_CREATED)

    def patch(self, request, *args, **kwargs):
        serializer = UserSerializer(self.get_object(), data=request.data, partial=True)
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return JSONResponse(serializer.data)

    @decorators.detail_route()
    def me(self, request):
        serializer = UserSerializer(request.user)
        return JSONResponse(serializer.data)

    @decorators.detail_route(required_scopes=['write'])
    def partial_me(self, request, **kwargs):
        serializer =UserSerializer(request.user, data=request.data, partial=True)
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return JSONResponse(serializer.data)

    # remove permissions for user creation.
    def get_permissions(self):
        if self.action == 'create':
            self.permission_classes = [permissions.AllowAny,]
        elif self.action in ['me', 'partial_me']:
            self.permission_classes = [permissions.IsAuthenticated]
        return super(self.__class__, self).get_permissions()

user_list = UserViewSet.as_view({
    'get': 'list',
    'post': 'create'
})

user_me = UserViewSet.as_view({
    'get': 'me',
    'patch': 'partial_me'
})

user_detail = UserViewSet.as_view({
    'get': 'retrieve',
    'patch': 'patch',
    'delete': 'destroy'
})

urlpatterns = [
    url(r'^$', user_list, name='user-list'),
    url(r'^me/', user_me, name='user-me'),
    url(r'^(?P<pk>[0-9]+)', user_detail, name='user-detail')
]


import numpy as np
import cv2
import pdb
from config import cfg

dst_threshold = cfg.DST_THR
reward_terminal_action = cfg.reward_terminal_action
reward_movement_action = cfg.reward_movement_action
reward_invalid_movement_action = cfg.reward_invalid_movement_action
reward_remove_action = cfg.reward_remove_action
# Different actions that the agent can do
num_of_actions = cfg.ACT_NUM
# Actions captures in the history vector
num_of_history = cfg.HIS_NUM

def get_dst(gt_point, cur_point):
    dst = abs(gt_point-cur_point)
    return dst

def get_reward_trigger(cur_dst):
    if cur_dst < dst_threshold:
        reward = reward_terminal_action
    else:
        reward = - reward_terminal_action
    return reward

def get_reward_movement(cur_point, last_point, gt_point):
    if gt_point == -100:
        if cur_point == -100:
            reward = reward_movement_action
        else:
            reward = - reward_movement_action
    else:
        cur_dst = get_dst(gt_point, cur_point)
        last_dst = get_dst(gt_point, last_point)
        if cur_dst < last_dst:
            reward = reward_movement_action
        else:
            reward = - reward_movement_action
    return reward

def getRewMov0427(cur_point, last_point, gt_point):
    if gt_point == -20:  # should be removed, but the action is 2 or 3
        reward = - reward_remove_action
    else:  # should be moved, the action is 2 or 3
        cur_dst = get_dst(gt_point, cur_point)
        last_dst = get_dst(gt_point, last_point)
        if cur_dst < last_dst:
            reward = reward_movement_action
        else:
            if cur_point < 0 or cur_point >= 100:  # moved out of the image, the reward is change to -5
                reward = reward_invalid_movement_action
            else:
                reward = - reward_movement_action
    return reward

def getRewRm(cur_dst):
    if cur_dst == 0:
        reward = reward_remove_action
    else:
        reward = - reward_remove_action
    return reward

def update_history_vector(history_vector, action):
    action_vector = np.zeros(num_of_actions)
    action_vector[action-1] = 1
    # number of real history in the current history vector
    num_real_cur_history = np.size(np.nonzero(history_vector))
    updated_history_vector = np.zeros(num_of_actions*num_of_history)
    if num_real_cur_history < num_of_history:
        aux2 = 0
        for l in range(num_of_actions*num_real_cur_history, num_of_actions*num_real_cur_history+num_of_actions):
            history_vector[l] = action_vector[aux2]
            aux2 += 1
        return history_vector
    else:
        for j in range(0, num_of_actions*(num_of_history-1)):
            updated_history_vector[j] = history_vector[j+num_of_actions]
        aux = 0
        for k in range(num_of_actions*(num_of_history-1), num_of_actions*num_of_history):
            updated_history_vector[k] = action_vector[aux]
            aux += 1
        return updated_history_vector

def get_state(cur_point, hist_vec):
    history_vector = np.reshape(hist_vec, (num_of_actions*num_of_history, 1))
    state = np.vstack((cur_point, history_vector))
    state = np.squeeze(state)
    return state


# coding=utf-8

from sqlalchemy import Column, String, Integer, Date, DateTime, BigInteger, Boolean, JSON, ARRAY, ForeignKey, Float
from sqlalchemy.orm import relationship

from db.base import Base
from history_fetcher.utils import get_prop

class StageStatisticsEntity(Base):
    """
    A class used to represent the stage_statistics entity in the database.

    Each stage in each Spark application should be represented by one record in the stage_statistics table.
    All the attributes (except for stage_key) should contain an array of five values, one for each quantile:
    0.001, 0.25, 0.5, 0.75, 0.999
    """
    __tablename__ = 'stage_statistics'

    stage_key = Column(String, ForeignKey('stage.stage_key'), primary_key=True)
    stage = relationship("StageEntity", back_populates="stage_statistics")
    quantiles = Column(ARRAY(Float))
    executor_deserialize_time = Column(ARRAY(Float))
    executor_deserialize_cpu_time = Column(ARRAY(Float))
    executor_run_time = Column(ARRAY(Float))
    executor_cpu_time = Column(ARRAY(Float))
    result_size = Column(ARRAY(Float))
    jvm_gc_time = Column(ARRAY(Float))
    result_serialization_time = Column(ARRAY(Float))
    getting_result_time = Column(ARRAY(Float))
    scheduler_delay = Column(ARRAY(Float))
    peak_execution_memory = Column(ARRAY(Float))
    memory_bytes_spilled = Column(ARRAY(Float))
    disk_bytes_spilled = Column(ARRAY(Float))
    bytes_read = Column(ARRAY(Float))
    records_read = Column(ARRAY(Float))
    bytes_written = Column(ARRAY(Float))
    records_written = Column(ARRAY(Float))
    shuffle_read_bytes = Column(ARRAY(Float))
    shuffle_read_records = Column(ARRAY(Float))
    shuffle_remote_blocks_fetched = Column(ARRAY(Float))
    shuffle_local_blocks_fetched = Column(ARRAY(Float))
    shuffle_fetch_wait_time = Column(ARRAY(Float))
    shuffle_remote_bytes_read = Column(ARRAY(Float))
    shuffle_remote_bytes_read_to_disk = Column(ARRAY(Float))
    shuffle_total_blocks_fetched = Column(ARRAY(Float))
    shuffle_write_bytes = Column(ARRAY(Float))
    shuffle_write_records = Column(ARRAY(Float))
    shuffle_write_time = Column(ARRAY(Float))

    def __init__(self, attributes):
        """
        Create a StageStatistics object.
        :param attributes: dictionary {name: value} containing the attributes
        """
        self.stage_key = get_prop(attributes, "stage_key")
        self.quantiles = get_prop(attributes, "quantiles")
        self.executor_deserialize_time = get_prop(attributes, "executor_deserialize_time")
        self.executor_deserialize_cpu_time = get_prop(attributes, "executor_deserialize_cpu_time")
        self.executor_run_time = get_prop(attributes, "executor_run_time")
        self.executor_cpu_time = get_prop(attributes, "executor_cpu_time")
        self.result_size = get_prop(attributes, "result_size")
        self.jvm_gc_time = get_prop(attributes, "jvm_gc_time")
        self.result_serialization_time = get_prop(attributes, "result_serialization_time")
        self.getting_result_time = get_prop(attributes, "getting_result_time")
        self.scheduler_delay = get_prop(attributes, "scheduler_delay")
        self.peak_execution_memory = get_prop(attributes, "peak_execution_memory")
        self.memory_bytes_spilled = get_prop(attributes, "memory_bytes_spilled")
        self.disk_bytes_spilled = get_prop(attributes, "disk_bytes_spilled")
        self.bytes_read = get_prop(attributes, "bytes_read")
        self.records_read = get_prop(attributes, "records_read")
        self.bytes_written = get_prop(attributes, "bytes_written")
        self.records_written = get_prop(attributes, "records_written")
        self.shuffle_read_bytes = get_prop(attributes, "shuffle_read_bytes")
        self.shuffle_read_records = get_prop(attributes, "shuffle_read_records")
        self.shuffle_remote_blocks_fetched = get_prop(attributes, "shuffle_remote_blocks_fetched")
        self.shuffle_local_blocks_fetched = get_prop(attributes, "shuffle_local_blocks_fetched")
        self.shuffle_fetch_wait_time = get_prop(attributes, "shuffle_fetch_wait_time")
        self.shuffle_remote_bytes_read = get_prop(attributes, "shuffle_remote_bytes_read")
        self.shuffle_remote_bytes_read_to_disk = get_prop(attributes, "shuffle_remote_bytes_read_to_disk")
        self.shuffle_total_blocks_fetched = get_prop(attributes, "shuffle_total_blocks_fetched")
        self.shuffle_write_bytes = get_prop(attributes, "shuffle_write_bytes")
        self.shuffle_write_records = get_prop(attributes, "shuffle_write_records")
        self.shuffle_write_time = get_prop(attributes, "shuffle_write_time")

    @staticmethod
    def get_attributes(stage_key, stage_statistics):
        """
        Get stage_statistics entity attributes as a key-value dict
        :param stage_key: stage key (string)
        :param stage_statistics: stage_statistics data (json)
        :return: dict (attribute: value)
        """
        return {
            'stage_key': stage_key,
            'quantiles': stage_statistics["quantiles"],
            'executor_deserialize_time': stage_statistics["executorDeserializeTime"],
            'executor_deserialize_cpu_time': stage_statistics["executorDeserializeCpuTime"],
            'executor_run_time': stage_statistics["executorRunTime"],
            'executor_cpu_time': stage_statistics["executorCpuTime"],
            'result_size': stage_statistics["resultSize"],
            'jvm_gc_time': stage_statistics["jvmGcTime"],
            'result_serialization_time': stage_statistics["resultSerializationTime"],
            'getting_result_time': stage_statistics["gettingResultTime"],
            'scheduler_delay': stage_statistics["schedulerDelay"],
            'peak_execution_memory': stage_statistics["peakExecutionMemory"],
            'memory_bytes_spilled': stage_statistics["memoryBytesSpilled"],
            'disk_bytes_spilled': stage_statistics["diskBytesSpilled"],
            'bytes_read': stage_statistics["inputMetrics"]["bytesRead"],
            'records_read': stage_statistics["inputMetrics"]["recordsRead"],
            'bytes_written': stage_statistics["outputMetrics"]["bytesWritten"],
            'records_written': stage_statistics["outputMetrics"]["recordsWritten"],
            'shuffle_read_bytes': stage_statistics["shuffleReadMetrics"]["readBytes"],
            'shuffle_read_records': stage_statistics["shuffleReadMetrics"]["readRecords"],
            'shuffle_remote_blocks_fetched': stage_statistics["shuffleReadMetrics"]["remoteBlocksFetched"],
            'shuffle_local_blocks_fetched': stage_statistics["shuffleReadMetrics"]["localBlocksFetched"],
            'shuffle_fetch_wait_time': stage_statistics["shuffleReadMetrics"]["fetchWaitTime"],
            'shuffle_remote_bytes_read': stage_statistics["shuffleReadMetrics"]["remoteBytesRead"],
            'shuffle_remote_bytes_read_to_disk': stage_statistics["shuffleReadMetrics"]["remoteBytesReadToDisk"],
            'shuffle_total_blocks_fetched': stage_statistics["shuffleReadMetrics"]["totalBlocksFetched"],
            'shuffle_write_bytes': stage_statistics["shuffleWriteMetrics"]["writeBytes"],
            'shuffle_write_records': stage_statistics["shuffleWriteMetrics"]["writeRecords"],
            'shuffle_write_time': stage_statistics["shuffleWriteMetrics"]["writeTime"]
        }


import os
import shutil
import sys
import test.util as util

# CLEAN       is the equivalent of running `make clean`.
# RESET       establishes a new source tree, repatching and reconfiguring.
# CACHE_CLEAN removes source tarballs.
# DIST_CLEAN  removes all source trees and downloaded data

CLEAN, RESET, CACHE_CLEAN, DIST_CLEAN = range(4)
OPT_0, OPT_1, OPT_2, OPT_3            = range(4)

default_opts = [(OPT_0, True), (OPT_2, False)]

class IntegrationTest(object):
  def __init__(self, arches, verbose, opts=default_opts):
    self.verbose = verbose
    self.arches = arches
    self.opts = opts
    self.local_tar = self.cwd(self.local_tar)

  def clean(self, level):
    pass

  def cwd(self, *args):
    return os.path.join(os.path.abspath(os.curdir),
                        'test',
                        'integration',
                        self.name(),
                        *args)

  def ref_dir(self, arch, opt, *args):
    return self.cwd('%s_%s_opt%d_ref' % (self.name(), arch, opt), *args)

  def proto_dir(self, arch, opt, *args):
    return self.cwd('%s_%s_opt%d_proto' % (self.name(), arch, opt), *args)

  def config(self):
    util.mkdir_p(self.cwd())

  def fetch(self):
    pass

class TarballTest(IntegrationTest):
  def __init__(self, arches, verbose):
    super(TarballTest,self).__init__(arches, verbose)

  def clean(self, level):
    if level == CACHE_CLEAN or level == DIST_CLEAN:
      if os.path.exists(self.local_tar):
        os.remove(self.local_tar)
      if level == CACHE_CLEAN:
        return
    for arch in self.arches:
      for opt,debug in self.opts:
        if level == CLEAN:
          if os.path.exists(self.ref_dir(arch, opt, self.make_dir)):
            util.make(self.ref_dir(arch, opt, make_dir), 'clean')
          if os.path.exists(self.proto_dir(arch, opt, self.make_dir)):
            util.make(self.proto_dir(arch, opt, make_dir), 'clean')
        else:
          if os.path.exists(self.ref_dir(arch, opt)):
            shutil.rmtree(self.ref_dir(arch, opt))
          if os.path.exists(self.proto_dir(arch, opt)):
            shutil.rmtree(self.proto_dir(arch, opt))

  def config(self):
    super(TarballTest, self).config()
    util.download_to(self.tar_url, self.local_tar, self.tar_md5, verbose=self.verbose)
    for arch in self.arches:
      for opt,debug in self.opts:
        if not os.path.exists(self.ref_dir(arch, opt)):
          util.untar_to(self.local_tar, self.ref_dir(arch, opt), verbose=self.verbose)
          util.patch(self.patch(arch, opt, debug, False), self.ref_dir(arch, opt), 1, verbose=self.verbose)
        if not os.path.exists(self.proto_dir(arch, opt)):
          util.untar_to(self.local_tar, self.proto_dir(arch, opt), verbose=self.verbose)
          util.patch(self.patch(arch, opt, debug, True), self.proto_dir(arch, opt), 1, verbose=self.verbose)

  def build_dir(self, build_dir, force_build=False):
    bin_file = os.path.join(build_dir, self.bin_name)
    if force_build or not os.path.exists(bin_file):
      util.make(build_dir, *self.make_args)

  def build(self, force_build=False):
    for arch in self.arches:
      for opt, debug in self.opts:
        self.build_dir(self.ref_dir(arch, opt, self.make_dir), force_build)
        assert not util.bin_links_to(self.ref_dir(arch, opt, self.make_dir, self.bin_name),
                                     os.path.join(os.environ['PROTO_LIB_DIR'],
                                                  'libproto.so'))
        self.build_dir(self.proto_dir(arch, opt, self.make_dir), force_build)
        assert util.bin_links_to(self.proto_dir(arch, opt, self.make_dir, self.bin_name),
                                 os.path.join(os.environ['PROTO_LIB_DIR'],
                                              'libproto.so'))


import urllib.request
import urllib.error
import urllib.parse
import json
from arbitrage.markets.market import MarketBase

class HuobiBase(MarketBase):
    def __init__(self, currency, code, config=None):
        super().__init__(currency, config)
        self.code = code
        self.update_rate = 30

    def update_depth(self):
        #/market/depth?symbol=ethcny&type=step1
        #url = 'http://api.huobi.com/staticmarket/depth_%s_50.js' % self.code
        url = 'http://api.huobi.pro/market/depth?symbol=%s&type=step0' % self.code
        req = urllib.request.Request(url, headers={
            "Content-Type": "application/x-www-form-urlencoded",
            "Accept": "*/*",
            "User-Agent": "curl/7.24.0 (x86_64-apple-darwin12.0)"})
        res = urllib.request.urlopen(req)
        depth = json.loads(res.read().decode('utf8'))['tick']
        self.depth = self.format_depth(depth)

    def sort_and_format(self, l, reverse=False):
        l.sort(key=lambda x: float(x[0]), reverse=reverse)
        r = []
        for i in l:
            r.append({'price': float(i[0]), 'amount': float(i[1])})
        return r

    def format_depth(self, depth):
        bids = self.sort_and_format(depth['bids'], True)
        asks = self.sort_and_format(depth['asks'], False)
        return {'asks': asks, 'bids': bids}


from imutils.video import VideoStream
import argparse
import time
import cv2
import simple_detect_barcode as bar_det

ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video",
	help="path to the (optional) video file")
args = vars(ap.parse_args())

if not args.get("video", False):
	vs = VideoStream(src=0).start()
	time.sleep(2.0)# eğer video yoksa webcami açar
else:
	vs = cv2.VideoCapture(args["video"])# video  verildi ise video kullanılır

while True:

    frame = vs.read()
    frame =frame[1] if args.get("video",False) else frame  #video okundu

    if frame is None:# vide bittiyse durdurur
        break

    box=bar_det.detect(frame)

    if box is not None :# barcode tespit edildi ise konturun etrafını çizer
          cv2.drawContours(frame,[box],-1,(0,255,0),2)

    cv2.imshow("Frame", frame)
    key = cv2.waitKey(1) & 0xFF

    if key == ord("q"):
        break

if not args.get("video", False):# video okuma bittiyse  durdur
    vs.stop()
else:
    vs.release()# ya da webcami bırak

cv2.destroyAllWindows()


import sys
input = sys.stdin.readline # sys.stdin.readline 을 번거롭게 계속 써주는 것을 피하기 위해 input 변수를 사용하였다.

n = int(input()) # 일반적인 input() 함수가 아니라 sys.stdin.readline()이다. input 변수를 sys.stdin.readline()로 지정해주었기 때문이다.
card_dic = {} # 이 문제에서 딕셔너리를 활용했다. 딕셔너리의 장점은 key와 value를 같이 묶어서 활용할 수 있다는 것이다.

for i in range(n) :
    card = int(input()) # 카드의 정수 값을 입력받는다.
    if card in card_dic :
        card_dic[card] += 1 # 딕셔너리에 한 번이상은 집어넣은 카드의 정수값은 value만 1 올려준다.
    else :
        card_dic[card] = 1 # 카드의 정수값을 key로 설정한 후, 정수값 개수를 value로 설정하여 딕셔너리에 집어넣는다.

result = sorted(card_dic.items(),key = lambda x : (-x[1],x[0]))
# sorted(정렬할 데이터, key 파라미터)
# key 파라미터: 어떤 것을 기준으로 정렬할 것인가?에 대한 기준. 즉, 기준을 정해 줄 수 있는 파라미터
# lambda 인자(매개변수):표현식(인자를 이용한 동작) : 일반함수를 가볍게 만든 함수라고 생각하자. ex) lambda x:x*2
# 여기서는 lambda가 key 파라미터를 위해 쓰였다. 즉, 기준을 설정하기 위해 쓰였는데, 그 기준을 value(x[1])는 내림차순(-)으로 , key(x[0])는 오름차순(+)으로 설정한 것이다. 그리고 이를 result에 담는다.
print(result[0][0])
# result[0][0] -> 첫번째 [0]는 첫번째 item을 나타내는 것이고, 두번째 [0]는 첫번째 item의 첫번째 요소, 즉 정수 값이다. 그리고 이는 가장 많은 개수인 정수 값 중 가장 작은 정수이다.

개인 실습/알고리즘 문제/백준 11652 카드.py


#Given a non-empty array of integers nums, every element appears twice except for one. Find that single one.
nums = [4,1,2,1,2,3,4,5,3]

class Solution:
    def singleNumber(self, nums) -> int:

        # result = 0
        # for i in arr:
        #     result ^= i
        # return result
        #Bit manipulation technique
        # refer https://www.google.com/search?q=++Single+Number+python&client=firefox-b-d&ei=1r6HZJL5PK6MseMPxOClgA4&ved=0ahUKEwiSvPD9hL__AhUuRmwGHURwCeAQ4dUDCA4&uact=5&oq=++Single+Number+python&gs_lcp=Cgxnd3Mtd2l6LXNlcnAQAzIFCAAQgAQyBggAEBYQHjIGCAAQFhAeMgYIABAWEB4yBggAEBYQHjIGCAAQFhAeMgYIABAWEB4yBggAEBYQHjIGCAAQFhAeMgYIABAWEB46CggAEEcQ1gQQsAM6BwgAEIAEEAo6CAgAEIoFEIYDSgQIQRgAUNoDWJsQYLESaANwAXgAgAFuiAGVBpIBAzUuM5gBAKABAqABAcABAcgBCA&sclient=gws-wiz-serp#fpstate=ive&vld=cid:36e9b9b8,vid:qMPX1AOa83k
        # xor two values that are same and compare

        print(set(nums))
        print(2*sum(set(nums)))

        return 2*sum(set(nums)) - sum(nums)

result = Solution().singleNumber(nums)
print(result)


import numpy as np
from sklearn.model_selection import TimeSeriesSplit

X = np.array([[1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4]])
y = np.array([1, 2, 3, 4, 5, 6])
tscv = TimeSeriesSplit(n_splits=3)
print ("tscv: {}".format(tscv))

for train_index, test_index in tscv.split(X):
    print ("{} {}".format(train_index, test_index))


'''
Optimized version of sentiment_run. Should work better on larger datsets. However, I would not recommend choosing more
than 300 wacrcs to analyze, as it will take a long time to run.

For this script to work, a model needs to be in the models folder and it needs to be named model_dl. Training a model is advanced.
If you want to train a model, please use the training_fin_classifier.py script, but you can also ask me and I will share the model
on google drive. (too large to upload to github)

Usuage:
0. Start a cluster with spark-dev cells
1. run sentiment_long_boot with ip as parameter and pem location and seconr parameter
example: sh sentiment_long_boot.sh ec2-54-160-226-58.compute-1.amazonaws.com "/home/johan/Johan_key.pem"
2.ssh onto master node and mkdir articlespar.parquet, then sudo docker build -t sentiment_long_boot . and run the docker build
with sudo docker run -e RANSAMPLE=y -e NUMWARCS=120 -e NUMRECORDS=200 -e PYTHONFILETORUN=./sentiment_long_optim.py sentiment_long_boot
if running locally just build and run dockerfile but with additional arguments of (-e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY=)

you can pass -e RANSAMPLE=y to randomly sample the crawls, or -e RANSAMPLE=n to take all the most recent crawls.
you can pass -e NUMRECORDS=100 to change the number of records to extract from each crawl.
you can pass -e NUMWARCS=1 to change the number of crawls to analyze.

'''

from pyspark.sql import SparkSession, DataFrame
from pyspark.sql.types import StructType, StructField, StringType, LongType, IntegerType
import sparknlp
spark = sparknlp.start()
# sparknlp.start(gpu=True) >> for training on GPU
from sparknlp.base import *
from sparknlp.annotator import *
from pyspark.ml import Pipeline
from langdetect import detect
from pyspark.sql.functions import col, lit, concat_ws

from sklearn.metrics import classification_report
import requests
from warcio import ArchiveIterator
from bs4 import BeautifulSoup
import time
import pandas as pd
import re
import yfinance as yf
import boto3
import random
import sys
import numpy as np
import os
#PARAMETERS
path_dl_model = './models/model_dl'
batch_size_max = sys.maxsize -1
num_records_percrawl = int(os.environ['NUMRECORDS']) #number of recors to attempt to extract from each crawl
ticker = 'SPY'
#read in financewordlist.csv into the list
wordlist = pd.read_csv('./sentdat/topics.csv', header=None)[0].tolist()
wordlist.extend(yf.Ticker(ticker).info['longName'].split())
number_warcs_to_analyze = int(os.environ['NUMWARCS']) #number of warcs to perform sentiment analysis on, goes from most reccent to farther back onse
randomsample = str(os.environ['RANSAMPLE']).lower() #Y or N, if Y, then it will take a random sample of warcs to analyze, if N, it will take the most recent warcs
#CREATING THE PIPELINE FOR LATER
document_assembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

sentence_detector = SentenceDetector() \
    .setInputCols(["document"]) \
    .setOutputCol("sentence")

tokenizer = Tokenizer() \
    .setInputCols(['sentence']) \
    .setOutputCol('token')

lemmatizer = Lemmatizer()\
    .setInputCols(['token'])\
    .setOutputCol('lemma')\
  .setDictionary("./sentdat/lemmas_small.txt", key_delimiter="->", value_delimiter="\t")
SentimentDetector = sentiment.SentimentDetector() \
    .setInputCols(['lemma', 'sentence'])\
    .setOutputCol('sentiment_score')\
    .setDictionary('./sentdat/sentiment-big.csv', ',')\

pipeline = Pipeline(stages=[
    document_assembler,
    sentence_detector,
    tokenizer,
    lemmatizer,
    SentimentDetector
])
###GETTING WARC FILE NAMES FROM S3, GRABBING A RANDOM SAMPLE OF THEM
s3 = boto3.resource('s3')
my_bucket = s3.Bucket('commoncrawl')
warcs = []
for object in my_bucket.objects.filter(Prefix='crawl-data/CC-NEWS/'):
    if object.key.endswith('.warc.gz'):
        warcs.append(object.key)

if randomsample == 'y':
    warcs = random.sample(warcs, number_warcs_to_analyze)
else:
  warcs = warcs[-number_warcs_to_analyze:]

for index, warc in enumerate(warcs):
    warcs[index] = 'https://data.commoncrawl.org/' + warc

#function to convert time from commoncrawl format to y-m-d
def convert_header_date(date):
    return time.strftime('%Y-%m-%d', time.strptime(date, '%Y-%m-%dT%H:%M:%SZ'))

#obtaining stock data from yahoo finance from 2019 to current date.
currentdate = time.strftime("%Y-%m-%d")
stockdata = yf.download(ticker, start='2010-01-01', end=currentdate)['Adj Close']

#creating scehma to store text and prices
data = StructType([\
  StructField("text", StringType(), True),
    StructField("price", StringType(), True),
    StructField("date", StringType(), True)
]
)
print('starting to load model')
FINDMODEL = PipelineModel.load(path_dl_model)
print('model loaded')

#function to drop non-finance articles
def drop_nonfinance_articles(df):
  df = FINDMODEL.transform(df)
  df = df.withColumn('finance', df['financial_model_pred.result'].getItem(0).cast('float'))
  df = df.filter(df['finance'] == 1.0)
  return df

batching_done = False
pausing_index = 0
while batching_done == False:
  # creating the main rdd to store the data
  list_of_rows_batch = []
  rows_batch_len = 0
  recordsfetched = 0
  failures = 0
  datelist = []

  for index, warc_url in enumerate(warcs):
      response = requests.get(warc_url, stream=True)
      if response.ok!=True:
          raise Exception("Error downloading WARC file")
      records = ArchiveIterator(response.raw, arc2warc=True)
      #what this should do is write each record's plaintexxt to a csv file
      for record in records:
          if record.rec_type == 'response':
              try:
                  html = record.content_stream().read() .decode('utf-8')
                  plaintext = BeautifulSoup(html, 'lxml').get_text()
                  plaintext = re.sub(r'\s+', ' ', plaintext)
                  plaintext = re.sub(r'[^a-zA-Z0-9\s]', '', plaintext).lower()

                  #obtains plaintext from the html
                  if detect(plaintext) == 'en' and len(plaintext) > 150:
                      date = record.rec_headers.get_header('WARC-Date')
                      date = convert_header_date(date)
                      # append the plaintext and price to the batch
                      if date in stockdata.index:
                          datelist.append(date)
                          list_of_rows_batch.append({'text':plaintext, 'price':float(stockdata[date]), 'date':date})
                          recordsfetched += 1
                          rows_batch_len += 1
                      else:
                          print('date not in stockdata',date)
                          #likely a weekend or holiday, so we will just skip the entire warc
                          break
                  else:
                      recordsfetched += 1
              except:
                  recordsfetched += 1  # because if the entire warc file is not in english or wrong date, we still want to move on to the next one
                  failures += 1
                  #print("attempt record: ", record.rec_headers.get_header('WARC-Target-URI'), " failed")
                  pass

          if rows_batch_len >= batch_size_max:
              print("batch size max reached")
              pausing_index = index

          if recordsfetched >= num_records_percrawl:
              recordsfetched = 0
              print("warc done")
              break

      #finishing up for the last batch in it wasn't full and num batches wasnt maxed out.
  if rows_batch_len > 0:
      datelist = np.unique(datelist)
      newdatelist = []
      for date in datelist:
         newdatelist.append(str(date))
      datelist = newdatelist
      print("done one batch, size: ", rows_batch_len)
      rows_batch_len = 0

  if pausing_index != 0:
      warcs = warcs[pausing_index:]
      pausing_index = 0
  else:
      batching_done = True

  print("failures: ", failures)

  #now split list_of_rows_batch by dates in datelist

  list_of_lists_freach_date = []
  for date in datelist:
      list_of_lists_freach_date.append([row for row in list_of_rows_batch if row['date'] == date])

  sentscores = []
  finacial_data = []

  for index, one_date_lst in enumerate(list_of_lists_freach_date):
    df = spark.createDataFrame(one_date_lst, schema=data)
    #dropping non-finance articles
    df = drop_nonfinance_articles(df)
    numarticles = df.count()
    print("dropped non-finance articles, num articles for this date: ", numarticles, " date: ", datelist[index])
    #drop unessecary columns created from dropping non-finance articles
    cols = df.columns
    for item in ['text', 'price', 'date']:
        cols.remove(item)
    df = df.drop(*cols)

    df = pipeline.fit(df).transform(df)

    df = df.withColumn("sentiment_score", concat_ws(",", "sentiment_score.result"))

    positives= df.filter(col('sentiment_score') == 'positive').count()
    negatives = numarticles - positives
    print("total positive and negatives for ", datelist[index])
    print("positives", positives)
    print("negatives", negatives)
    if negatives == 0:
      sentscores.append(positives)
    else:
      sentscores.append(positives/negatives)
    finacial_data.append(float(stockdata[datelist[index]]))

import matplotlib.pyplot as plt
x = np.arange(len(finacial_data))
plt.plot(x, finacial_data)
plt.xlabel('dates')
plt.ylabel('stock price')
plt.show()
plt.plot(x, sentscores)
plt.xlabel('dates')
plt.ylabel('sentiment score')
plt.show()
#normalize data
finacial_data_plt = (finacial_data - np.mean(finacial_data))/np.std(finacial_data)
sentscores_plt = (sentscores - np.mean(sentscores))/np.std(sentscores)
plt.plot(x, finacial_data_plt)
plt.plot(x, sentscores_plt)
plt.xlabel('dates')
plt.ylabel('normalized stock price and sentiment score')
plt.show()

print(sentscores)
print(finacial_data)
print(datelist)


import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf

from utils import *
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.callbacks import EarlyStopping, LambdaCallback

tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

df = pd.read_csv('data.csv', names=column_names)
df.head()
df.isna().sum()
df = df.iloc[1:, 1:]
df_norm = (df - df.mean()) / df.std()
df_norm.head()
y_mean = df['price'].mean()
y_std = df['price'].std()

def convert_label_value(pred):
    return int(pred * y_std + y_mean)  # trzeba cofnac normalizacje ceny

x = df_norm.iloc[:, :6]
x.head()
y = df_norm.iloc[:, -1]
y.head()
x_arr = x.values
y_arr = y.values
x_train, x_test, y_train, y_test = train_test_split(x_arr, y_arr, test_size=0.05, random_state=0)

def get_model():
    model = Sequential([
        Dense(10, input_shape=(6,), activation='relu'),
        Dense(15, activation='relu'),
        Dense(10, activation='relu'),
        Dense(5, activation='relu'),
        Dense(1)
    ])
    model.compile(
        loss='mse',  # mean square error
        optimizer='adam'
    )
    return model

get_model().summary()
es_cb = EarlyStopping(monitor='val_loss', patience=5)

model=get_model()
preds_on_untrained = model.predict(x_test)

history = model.fit(
x_train, y_train,
    validation_data = (x_test, y_test),
    epochs = 100,
    callbacks = [es_cb]
)

plot_loss(history)


import pyomo.common.unittest as unittest

from pyomo.environ import ConcreteModel, Var, Block, Param, Expression, Constraint, Objective,  ExternalFunction, Reals, log, sin, sqrt, expr
from pyomo.contrib.viewer.model_browser import ComponentDataItem
from pyomo.contrib.viewer.ui_data import UIData

class TestDataModelItem(unittest.TestCase):
    def setUp(self):
        # Borrowed this test model from the trust region tests
        m = ConcreteModel()
        m.z = Var(range(3), domain=Reals, initialize=2.)
        m.x = Var(range(4), initialize=2.)
        m.x[1] = 1.0
        m.x[2] = 0.0
        m.x[3] = None

        m.b1 = Block()
        m.b1.e1 = Expression(expr=m.x[0] + m.x[1])
        m.b1.e2 = Expression(expr=m.x[0]/m.x[2])
        m.b1.e3 = Expression(expr=m.x[3]*m.x[1])
        m.b1.e4 = Expression(expr=log(m.x[2]))
        m.b1.e5 = Expression(expr=log(m.x[2] - 2))

        def blackbox(a,b):
            return sin(a-b)
        self.bb = ExternalFunction(blackbox)

        m.obj = Objective(
            expr=(m.z[0]-1.0)**2 + (m.z[0]-m.z[1])**2 + (m.z[2]-1.0)**2 \
                + (m.x[0]-1.0)**4 + (m.x[1]-1.0)**6 # + m.bb(m.x[0],m.x[1])
            )
        m.c1 = Constraint(expr=m.x[0] * m.z[0]**2 + self.bb(m.x[0],m.x[1]) == 2*sqrt(2.0))
        m.c2 = Constraint(expr=m.z[2]**4 * m.z[1]**2 + m.z[1] == 8+sqrt(2.0))
        m.c3 = Constraint(expr=m.x[1] == 3)
        m.c4 = Constraint(expr=0 == 3/m.x[2])
        m.c5 = Constraint(expr=0 == log(m.x[2]))
        m.c6 = Constraint(expr=0 == log(m.x[2]-4))
        m.c7 = Constraint(expr=0 == log(m.x[3]))
        m.p1 = Param(mutable=True, initialize=1)
        m.c8 = Constraint(expr = m.x[1] <= 1/m.p1)
        m.p1 = 0
        self.m = m.clone()

    def test_expr_calc(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.b1.e1)
        cdi.ui_data.calculate_expressions()
        self.assertAlmostEqual(cdi.get("value"), 3)
        self.assertIsInstance(cdi.get("expr"), str) # test get expr str

    def test_expr_calc_div0(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.b1.e2)
        cdi.ui_data.calculate_expressions()
        self.assertEqual(cdi.get("value"), "Divide_by_0")

    def test_expr_calc_log0(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.b1.e4)
        cdi.ui_data.calculate_expressions()
        self.assertIsNone(cdi.get("value"))

    def test_expr_calc_log_neg(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.b1.e5)
        cdi.ui_data.calculate_expressions()
        self.assertIsNone(cdi.get("value"))

    def test_expr_calc_value_None(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.b1.e3)
        cdi.ui_data.calculate_expressions()
        self.assertIsNone(cdi.get("value"))

    def test_cons_calc(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c3)
        cdi.ui_data.calculate_constraints()
        self.assertAlmostEqual(cdi.get("residual"), 2)

    def test_cons_calc_div0(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c4)
        cdi.ui_data.calculate_constraints()
        self.assertEqual(cdi.get("value"), "Divide_by_0")

    def test_cons_calc_log0(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c5)
        cdi.ui_data.calculate_constraints()
        self.assertIsNone(cdi.get("value"))

    def test_cons_calc_log_neg(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c6)
        cdi.ui_data.calculate_constraints()
        self.assertIsNone(cdi.get("value"))

    def test_cons_calc_value_None(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c7)
        cdi.ui_data.calculate_constraints()
        self.assertIsNone(cdi.get("value"))

    def test_cons_calc_upper_div0(self):
        cdi = ComponentDataItem(
            parent=None, ui_data=UIData(model=self.m), o=self.m.c8)
        cdi.ui_data.calculate_constraints()
        # the ui lists the upper and lower attributes as ub and lb
        # this was originally so I could easily combine variables and
        # constarints in the same view, but I split them up, so may want
        # to reconsider that choise in the future. This is to remind myself
        # why I'm getting "ub" and not "upper"
        self.assertEqual(cdi.get("ub"), "Divide_by_0")

    def test_var_get_value(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        self.assertAlmostEqual(cdi.get("value"), 1)
        self.assertIsNone(cdi.get(expr)) #test can't get expr

    def test_var_get_bounds(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        self.m.x[1].setlb(0)
        self.m.x[1].setub(10)
        self.assertAlmostEqual(cdi.get("lb"), 0)
        self.assertAlmostEqual(cdi.get("ub"), 10)

    def test_var_set_bounds(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        cdi.set("lb", 2)
        cdi.set("ub", 8)
        self.assertAlmostEqual(cdi.get("lb"), 2)
        self.assertAlmostEqual(cdi.get("ub"), 8)

    def test_var_fixed_bounds(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        cdi.set("fixed", True)
        self.assertTrue(cdi.get("fixed"))
        cdi.set("fixed", False)
        self.assertFalse(cdi.get("fixed"))

    def test_get_attr_that_does_not_exist(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        self.assertIsNone(cdi.get("test_val"))

    def test_set_func(self):
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x[1])
        self.assertIsNone(cdi.set("test_val", 5))
        self.assertIsNone(cdi.get("test_val")) # test can't set
        cdi = ComponentDataItem(parent=None, ui_data=None, o=self.m.x)
        self.assertIsNone(cdi.set("test_val", 5))
        self.assertEqual(cdi.get("test_val"), 5) # test can set with no callback

    def test_degrees_of_freedom(self):
        import pyomo.contrib.viewer.report as rpt
        # this should hit everything in report.  It only exists to calculate
        # degrees of freedom for display in the ui
        self.assertEqual(rpt.degrees_of_freedom(self.m),0)


from PySide2.QtCore import QCoreApplication, QEventLoop, QThread
from betse.util.io.log import logs
from betse.util.py import pythread
from betse.util.type.types import type_check
from betsee.guiexception import BetseePySideThreadException
from betsee.util.type.guitype import QThreadOrNoneTypes

# ....................{ GLOBALS                           }....................
_THREAD_MAIN_ID = None
'''
Arbitrary Qt-specific integer uniquely identifying the main thread in the
active Python interpreter if the currently installed version of PySide2 exposes
the :meth:`QThread.currentThreadId` method yielding this integer *or* the
Python-specific ID of this thread's object wrapper instead.

Caveats
----------
This integer is non-portable and hence should *only* be embedded in low-level
debugging messages. Moreover, this integer is ``None`` until the :func:`init`
function is called. In other words, you probably want to try something else.
'''

# ....................{ EXCEPTIONS                        }....................
def die_if_thread_current_main() -> None:
    '''
    Raise an exception if the current thread is the main thread in the active
    Python interpreter.

    This function should typically be called for safety by callers attempting
    to subsequently block the current thread (e.g., by calling the
    :func:`wait_for_events_if_none` function). The main thread administers the
    Qt-specific GUI event loop and hence should *never* be blocked for any
    duration of time whatsoever.

    Raises
    ----------
    BetseePySideThreadException
        If this function is called from the main thread.

    See Also
    ----------
    :func:`is_thread_current_main`
        Further details.
    '''

    if is_thread_current_main():
        raise BetseePySideThreadException(QCoreApplication.translate(
            'die_if_thread_current_main',
            'Operation prohibited in main thread.'))

# ....................{ EXCEPTIONS ~ private              }....................
def _die_unless_thread_main_id() -> None:
    '''
    Raise an exception unless a prior call to the :func:`init` function has
    defined the :data:`_THREAD_MAIN_ID` global.

    Raises
    ----------
    BetseePySideThreadException
        If the :func:`init` function has yet to be called.
    '''

    if _THREAD_MAIN_ID is None:
        raise BetseePySideThreadException(QCoreApplication.translate(
            '_die_unless_thread_main_id',
            '"_THREAD_MAIN_ID" undefined (e.g., '
            'due to the init() function having yet to be called).'))

# ....................{ TESTERS                           }....................
@type_check
def is_thread_current_main() -> bool:
    '''
    ``True`` only if the current thread is the main thread in the active Python
    interpreter.
    '''

    # Raise an exception unless the "_THREAD_MAIN_ID" global is defined.
    _die_unless_thread_main_id()

    # Return true only if this thread's identifier is that of the main thread.
    return get_thread_current_id() == _THREAD_MAIN_ID

@type_check
def should_halt_thread_work(thread: QThreadOrNoneTypes = None) -> bool:
    '''
    ``True`` only if some object has requested that the passed thread of
    execution voluntarily halt *all* tasks (e.g., overriden :meth:`QThread.run`
    method) and workers (e.g., subclassed :class:`QBetseeThreadWorkerABC`
    object) currently running in this thread.

    This function is intended to be voluntarily called by tasks and workers.

    Caveats
    ----------
    One parent thread may run an arbitrary number of child workers. However,
    the request to halt tested by this function is a low-level condition
    applying to a single thread rather than a single worker. Hence, if this
    function returns ``False``, *all* tasks and workers currently running in
    this thread are expected to gracefully halt all work being performed and
    then terminate.

    For fine-grained control over worker lifecycles, external callers are
    strongly advised to signal each such worker to gracefully halt (e.g., by
    emitting a signal connected to the :meth:`QBetseeThreadWorkerABC.stop`
    slot) rather than requesting that the thread running those workers halt
    (e.g., by calling the :meth:`QThread.requestInterruption` method).
    Nonetheless, workers are expected to respect both types of requests.

    For convenience, note that the
    :meth:`QBetseeThreadWorkerABC._halt_work_if_requested` method implicitly
    respects both types of requests. Ergo, :class:`QBetseeThreadWorkerABC`
    subclasses need *not* and should *not* explicitly call this function.

    Parameters
    ----------
    thread : QThreadOrNoneTypes
        Thread to request to be halted. Defaults to ``None``, in which case the
        current thread of execution is requested to be halted.

    Returns
    ----------
    bool
        ``True`` only if some object has requested this thread to voluntarily
        halt *all* tasks and workers currently running in this thread.

    See Also
    ----------
    :func:`halt_thread_work`
        Companion function requesting this thread to voluntarily halt these
        tasks and workers.
    '''

    # Default this thread if unpassed to the current thread of execution.
    if thread is None:
        thread = get_thread_current()

    # Return true only if this thread has been requested to be halted.
    return thread.isInterruptionRequested()

# ....................{ GETTERS ~ current                 }....................
def get_thread_current() -> QThread:
    '''
    Wrapper encapsulating the **current thread** (i.e., Qt-specific thread of
    execution responsible for the call to this function).
    '''

    return QThread.currentThread()

# Placeholder function definition to avoid spurious linter errors. For
# efficiency, this frequently called function is conditionally defined by the
# init() function at application initialization time.
def get_thread_current_id() -> int:
    pass

def get_thread_current_process_name() -> str:
    '''
    Name of the low-level kernel-level process associated with the current
    thread, equivalent to the Qt-specific name of this object.

    See Also
    ----------
    :meth:`betsee.util.thread.guithreadcls.QBetseeWorkerThread.process_name`
        Further details.
    '''

    # Current thread.
    thread = get_thread_current()

    # Return this thread's process name.
    return thread.objectName()

# ....................{ GETTERS ~ current : event         }....................
#FIXME: This submodule is increasingly awkward, largely due to the unctuous
#requirement that a "thread" reference always be passed whenever an
#"event_dispatcher" reference is required. Instead, do the following:
#
#* Define a new "ThreadEventDispatcherWrapper" class subclassing "object"
#  rather than "QObject" in this submodule.
#* Define the following instance variables of this class:
#  * "thread".
#  * "event_dispatcher".
#* Redefine the process_thread_events() and wait_for_events_if_none() functions into
#  methods of this class operating upon these variables.
#* Redefine the get_event_dispatcher_current() to return an instance of the
#  "ThreadEventDispatcherWrapper" class.
def get_thread_event_dispatcher_current() -> tuple:
    '''
    2-tuple ``(thread, event_dispatcher)`` encapsulating both the thread *and*
    event dispatcher for the current thread of execution if this thread has an
    event dispatcher *or* raise an exception otherwise (i.e., if this thread
    has no event dispatcher).

    Design
    ----------
    This getter intentionally returns both the thread *and* event dispatcher
    for the current thread of execution rather than merely the latter. Why?
    Because it appears infeasible to pass around arbitrary instances of the
    :class:`QAbstractEventDispatcher` superclass. For unknown reasons, some
    combination of Python and/or Qt aggressively collect an event dispatcher in
    the absence of a strong reference to the thread of that event dispatcher.
    Ergo, the two *must* be passed around as a single, cohesive unit.

    Returns
    ----------
    (QThread, QAbstractEventDispatcher)
        2-tuple ``(thread, event_dispatcher)``, where:

        * ``thread`` is the Qt-based wrapper encapsulating the current thread.
        * ``event_dispatcher`` is the Qt-based wrapper for the event dispatcher
          of the current thread.

    Raises
    ----------
    BetseePySideThreadException
        If this thread has *no* event dispatcher.
    '''

    # Current thread and event dispatcher of this thread (if any).
    thread = QThread.currentThread()
    event_dispatcher = thread.eventDispatcher()

    # If this thread has *NO* event dispatcher, raise an exception.
    if not event_dispatcher:
        raise BetseePySideThreadException(QCoreApplication.translate(
            'get_thread_event_dispatcher_current',
            'No event dispatcher running for thread "{}".'.format(
                thread.objectName())))

    # Return this thread and event dispatcher as a single, cohesive unit.
    return thread, event_dispatcher

# ....................{ LOGGERS ~ current                 }....................
@type_check
def log_debug_thread_current(message: str, *args) -> None:
    '''
    Log the passed debug message from any arbitrary thread with the root
    logger, formatted with the passed ``%``-style positional arguments.

    Parameters
    ----------
    message : str
        Human-readable message to be logged.

    All remaining positional parameters are conditionally interpolated as
    ``%``-style format specifiers into this message by logging facilities.

    See Also
    ----------
    :func:`log_debug_thread_main`
        Faster analogue intended to be called *only* when the current thread is
        the main thread.
    '''

    # Log this message at the debug level. See log_main_thread_debug().
    logs.log_debug(
        message + ' <from thread "%d">', *args, get_thread_current_id())

@type_check
def log_warning_thread_current(message: str, *args) -> None:
    '''
    Log the passed warning message from any arbitrary thread with the root
    logger, formatted with the passed ``%``-style positional arguments.

    Parameters
    ----------
    message : str
        Human-readable message to be logged.

    All remaining positional parameters are conditionally interpolated as
    ``%``-style format specifiers into this message by logging facilities.
    '''

    # Log this message at the warning level. See log_main_thread_debug().
    logs.log_warning(
        message + ' <from thread "%d">', *args, get_thread_current_id())

# ....................{ LOGGERS ~ main                    }....................
@type_check
def log_debug_thread_main(message: str, *args) -> None:
    '''
    Log the passed debug message from the main thread with the root logger,
    formatted with the passed ``%``-style positional arguments.

    Caveats
    ----------
    This function is intended to be called *only* from the main thread. For
    efficiency, this function does *not* validate this to be the case.

    Parameters
    ----------
    message : str
        Human-readable message to be logged.

    All remaining positional parameters are conditionally interpolated as
    ``%``-style format specifiers into this message by logging facilities.

    See Also
    ----------
    :func:`log_debug_thread_current`
        Slower analogue intended to be called from *any* thread.
    '''

    # Log this message at the debug level. To ensure that "%"-style format
    # specifiers embedded in this message are properly deferred until logging
    # time rather than performed now, the desired log message is created with
    # simple string concatenation rather than deferred until logging time with
    # "%"-style format specifiers, which would conflict with the former.
    logs.log_debug(
        message + ' <from main thread "%d">', *args, _THREAD_MAIN_ID)

# ....................{ HALTERS                           }....................
@type_check
def halt_thread_work(thread: QThreadOrNoneTypes = None) -> bool:
    '''
    Requested that the passed thread of execution voluntarily halt *all* tasks
    (e.g., overriden :meth:`QThread.run` method) and workers (e.g., subclassed
    :class:`QBetseeThreadWorkerABC` object) currently running in this thread.

    This function is intended to be called from objects in other threads.

    Caveats
    ----------
    After this function is called, this request will be unconditionally
    preserved in this thread until this thread is manually halted and restarted
    (e.g., by calling the :meth:`QBetseeWorkerThread.halt` and
    :meth:`QBetseeWorkerThread.start` methods in that order). Sadly, doing so:

    * Terminates all tasks and workers currently running in this thread,
      typically in a non-graceful manner resulting in in-memory or on-disk data
      corruption.
    * Discards all pending events currently queued with this thread --
      including both outgoing signals emitted by *and* incoming slots signalled
      on any workers still running in this thread.

    Hence, due to long-standing deficiencies in the Qt API, this request
    *cannot* be gracefully "undone." Attempting to do so *always* runs a risk
    of non-gracefully terminating running and pending work. The only
    alternative to this extremely concerning caveat is to signal each such
    worker to gracefully halt (e.g., by emitting a signal connected to the
    :meth:`QBetseeThreadWorkerABC.stop` slot) rather than calling this method.

    **You have been warned.** There be vicious vipers scuttling about here.

    Parameters
    ----------
    thread : QThreadOrNoneTypes
        Thread to request to be halted. Defaults to ``None``, in which case the
        current thread of execution is requested to be halted.

    See Also
    ----------
    :func:`should_halt_thread_work`
        Further details.
    https://forum.qt.io/topic/43067/qthread-requestinterruption-cannot-be-undone
        Prominent Qt forum discussion on this lamentable topic.
    '''

    # Default this thread if unpassed to the current thread of execution.
    if thread is None:
        thread = get_thread_current()

    # Request this thread's work to be halted.
    return thread.requestInterruption()

# ....................{ EVENTS ~ current                  }....................
@type_check
def process_thread_current_events() -> None:
    '''
    Process all pending events queued with the current thread.

    This function unconditionally processes *all* such events, including both
    outgoing signals emitted by *and* incoming slots signalled on all workers
    currently running in this thread.

    Caveats
    ----------
    Avoid manually calling the static :func:`QCoreApplication.processEvents`
    function to process events for the current thread of execution. If this
    thread has *no* event dispatcher, that function silently and hence unsafely
    ignores all pending :class:`DeferredDelete` events currently queued for
    this thread. Since numerous standard widgets (e.g., :class:`QToolTip`)
    require these events to be processed in a timely manner, ignoring these
    events fundamentally disrupts widget sanity.
    '''

    # Current thread and event dispatcher of this thread.
    thread, event_dispatcher = get_thread_event_dispatcher_current()

    # Process all pending events currently queued with this dispatcher.
    event_dispatcher.processEvents(QEventLoop.AllEvents)

@type_check
def wait_for_thread_current_event_if_none() -> None:
    '''
    Indefinitely block the current thread until *another* thread posts a new
    event to the current thread if *no* pending events are queued with the
    current thread *or* safely return immediately otherwise (i.e., if one or
    more pending events are currently queued with this thread).

    This function is intended to be called *only* from the slots of workers
    currently running in the thread of execution associated with the passed
    event dispatcher. In this case, this function call gracefully returns when
    another object in another thread of execution emits a signal connected to a
    slot of such a worker.

    Caveats
    ----------
    **Never call this function from an object owned by the main event-handling
    thread.** Doing so effectively blocks the application GUI managed by this
    thread. Instead, only call this function from a worker guaranteed to be
    running in this thread.
    '''

    # If the current thread is the main thread, raise an exception. The main
    # thread should *NEVER* be blocked for any period of time whatsoever.
    die_if_thread_current_main()

    # Current thread and event dispatcher of this thread.
    thread, event_dispatcher = get_thread_event_dispatcher_current()

    # Process all pending events currently queued with this dispatcher.
    event_dispatcher.processEvents(QEventLoop.WaitForMoreEvents)

# ....................{ INITIALIZERS                      }....................
def init() -> None:
    '''
    Initialize this submodule.

    Caveats
    ----------
    **This function is intended to be called from the main thread.** If this is
    *not* the case, the :data:`_THREAD_MAIN_ID` global will be improperly
    initialized.

    Raises
    ----------
    BetsePyException
        If the active Python interpreter does *not* support multithreading.
    '''

    # Submodule attributes at global scope (re)defined below. Yes, you declare
    # a function at global scope from within another function via the same
    # "global" declarator; failing to do so results in the former function
    # being declared as a locally scoped closure, instead. See also:
    #     https://stackoverflow.com/a/27930100/2809027
    global _THREAD_MAIN_ID, get_thread_current_id

    # Log this initialization.
    logs.log_debug('Initializing multithreading facilities...')

    # If the active Python interpreter does *NOT* support multithreading, raise
    # an exception *AFTER* BETSE is initialized.
    #
    # While Qt-based multithreading primitives are typically preferable to
    # Python-based multithreading primitives, the latter have their occasional
    # use. More critically, the lack of latter typically implies this
    # interpreter to be in insufficiently sane for our purposes.
    pythread.die_unless_threadable()

    # Conditionally define the frequently called get_thread_current_id() getter
    # by deciding whether PySide2 provides a binding for the static
    # QThread.currentThreadId() method on the current system and adopting the
    # best implementation available.
    #
    # Although Qt guarantees this method to exist, PySide2 bindings make no
    # such guarantees. For unknown reasons, all recent releases of PySide2 have
    # reliably failed to provide a binding for this method. Yes, this is a bug.
    try:
        QThread.currentThreadId
    # If PySide2 fails to expose this method, fallback to an implementation
    # returning the Python-specific ID of the object wrapper for this thread.
    except AttributeError:
        # Log this mishap as a non-fatal warning. What can we do, right?
        logs.log_warning(
            'QThread.currentThreadId() undefined; '
            'falling back to thread wrapper ID.')

        # Define this getter in the most hideous way possible.
        def get_thread_current_id() -> int:
            return id(get_thread_current())
    # Else, PySide2 exposes this method. In this case, adopt the preferred
    # implementation of simply deferring to this method.
    else:
        def get_thread_current_id() -> int:
            return QThread.currentThreadId()

    # Unconditionally document this getter regardless of implementation.
    get_thread_current_id.__doc__ = '''
    Arbitrary Qt-specific integer uniquely identifying the current thread in
    the active Python interpreter if the currently installed version of PySide2
    exposes the :meth:`QThread.currentThreadId` method yielding this integer
    *or* the Python-specific ID of this thread's object wrapper instead.

    Caveats
    ----------
    This integer is non-portable and hence should *only* be embedded in
    low-level debugging messages.

    Note that there exists no general-purpose :func:`get_thread_id` getter
    returning the unique identifier for an arbitrary thread, as Qt defines no
    corresponding functionality for doing so.

    See Also
    ----------
    :func:`get_thread_current`
        Further details.
    '''

    # Immediately identify the main thread (assumed without proof to be the
    # current thread) with the same function.
    _THREAD_MAIN_ID = get_thread_current_id()


"""Transformations using the well-known `Tseitin encoding
<https://en.wikipedia.org/wiki/Tseytin_transformation>`_.

The Tseitin transformation converts any arbitrary circuit to one in CNF in
polynomial time/space. It does so at the cost of introducing new variables
(one for each logical connective in the formula).
"""

from nnf import NNF, Var, And, Or, Internal
from nnf.util import memoize

def to_CNF(theory: NNF, simplify: bool = True) -> And[Or[Var]]:
    """Convert an NNF into CNF using the Tseitin Encoding.

    :param theory: Theory to convert.
    :param simplify: If True, simplify clauses even if that means eliminating
                     variables.
    """

    clauses = []

    @memoize
    def process_node(node: NNF) -> Var:

        if isinstance(node, Var):
            return node

        assert isinstance(node, Internal)

        children = {process_node(c) for c in node.children}

        if len(children) == 1:
            [child] = children
            return child

        aux = Var.aux()

        if simplify and any(~var in children for var in children):
            if isinstance(node, And):
                clauses.append(Or({~aux}))
            else:
                clauses.append(Or({aux}))

        elif isinstance(node, And):
            clauses.append(Or({~c for c in children} | {aux}))
            for c in children:
                clauses.append(Or({~aux, c}))

        elif isinstance(node, Or):
            clauses.append(Or(children | {~aux}))
            for c in children:
                clauses.append(Or({~c, aux}))

        else:
            raise TypeError(node)

        return aux

    @memoize
    def process_required(node: NNF) -> None:
        """For nodes that have to be satisfied.

        This lets us perform some optimizations.
        """
        if isinstance(node, Var):
            clauses.append(Or({node}))
            return

        assert isinstance(node, Internal)

        if len(node.children) == 1:
            [child] = node.children
            process_required(child)

        elif isinstance(node, Or):
            children = {process_node(c) for c in node.children}
            if simplify and any(~v in children for v in children):
                return
            clauses.append(Or(children))

        elif isinstance(node, And):
            for child in node.children:
                process_required(child)

        else:
            raise TypeError(node)

    process_required(theory)
    ret = And(clauses)
    NNF._is_CNF_loose.set(ret, True)
    NNF._is_CNF_strict.set(ret, True)
    return ret


from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^$', 'home.views.home', name='home'),
    url(r'^publications/', include('publications.urls')),
    url(r'^resources/', include('resources.urls')),
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^admin/', include(admin.site.urls)),
)


"""
Comparison of both bandpass measurements at frequencies above 183 GHz.

The SRF measurement of MWI-RX183_Matlab.xlsx is a xombination between the upper
and lower DSB measurements. Both sides are nearly weighted the same, but one
side can also be weighted more than the other side.

This script calculates MWI-RX183_Matlab.xlsx from the DSB measurements in the
Then, both measurements are compared by interpolation, because they measure not
at the same frequencies.

Outcome:
    Difference between both measurements is spectrally correlated for different
    channels. This can be seen clear when looking at differences in linear
    scale for overlapping bandpasses of channels 14 and 15, and 15 and 16.
    Correlation is higher for differences in dB (0.97, 0.92) than in linear
    values (0.85, 0.83) for channels 14/15 and 15/16 when using only the band
    pass region. When using all frequencies, the correlation is lower with
    values below 0.5.
"""

import numpy as np
import xarray as xr
import matplotlib.pyplot as plt
import os
from string import ascii_lowercase as abc
from dotenv import load_dotenv
from helpers import mwi, Sensitivity

plt.ion()
load_dotenv()

if __name__ == '__main__':

    sen1 = Sensitivity(filename=Sensitivity.files[0]).data
    sen2 = Sensitivity(filename=Sensitivity.files[1]).data

    #%% average sen1 double sideband sensitivity measurements
    freq_center = 183312  # where sen1 is mirrored

    sen1_left = sen1.lino.sel(
        frequency=sen1.frequency < freq_center).sortby(
            'frequency', ascending=False)
    sen1_left['frequency'] = 2*freq_center - sen1_left['frequency']

    sen1_right = sen1.lino.sel(frequency=sen1.frequency > freq_center)

    assert (sen1_left['frequency'] == sen1_right['frequency']).all()

    # add both sides and normalize to a sum of 1
    sen2_from_sen1 = sen1_left + sen1_right
    sen2_from_sen1 = sen2_from_sen1/sen2_from_sen1.sum('frequency')

    # interpolate sen2 on same frequencies as sen2_from_sen1
    sen2_interp = sen2.lino.interp({'frequency': sen2_from_sen1.frequency},
                                   method='linear')
    sen2_interp = sen2_interp/sen2_interp.sum('frequency')

    # combine in dataset
    ds = xr.Dataset()
    ds['sen2_from_sen1'] = sen2_from_sen1
    ds['sen2_interp'] = sen2_interp

    # convert to dB and shift to maximum of 0 dB
    for name in ['sen2_from_sen1', 'sen2_interp']:
        ds[name+'_dB'] = 10*np.log10(ds[name])
        ds[name+'_dB'] -= ds[name+'_dB'].max('frequency')

    assert ds[['sen2_from_sen1_dB', 'sen2_interp_dB']].max('frequency') == 0

    # calculate difference
    ds['sen2_diff'] = ds['sen2_interp'] - ds['sen2_from_sen1']
    ds['sen2_diff_dB'] = ds['sen2_interp_dB'] - ds['sen2_from_sen1_dB']

    assert (ds['sen2_diff'].sum('frequency') < 1e-15).all()

    # normalized difference
    ds['sen2_diff_norm'] = ds['sen2_diff'] / ds['sen2_interp']

    # correlation
    sen2_diff_corr = ds['sen2_diff'].to_pandas().corr()
    sen2_diff_dB_corr = ds['sen2_diff_dB'].to_pandas().corr()
    sen2_diff_norm = ds['sen2_diff_norm'].to_pandas().corr()

    # correlation only in bandpass region
    ds['sen2_diff_bp'] = ds['sen2_diff'].copy()
    ds['sen2_diff_dB_bp'] = ds['sen2_diff_dB'].copy()
    for i in range(5):
        ix = (ds.frequency > mwi.freq_bw_MHz[i, 2]) & \
            (ds.frequency < mwi.freq_bw_MHz[i, 3])
        ds['sen2_diff_bp'][:, i] = ds['sen2_diff_bp'][:, i].where(ix)
        ds['sen2_diff_dB_bp'][:, i] = ds['sen2_diff_dB_bp'][:, i].where(ix)

    sen2_diff_bp_corr = ds['sen2_diff_bp'].to_pandas().corr()
    sen2_diff_dB_bp_corr = ds['sen2_diff_dB_bp'].to_pandas().corr()

    # mean over bandpass region
    ds['sen2_diff_dB_bp'].mean('frequency')
    ds['sen2_diff_dB'].mean('frequency')

    #%% visualization: sensitivity graphs
    # linear
    fig, axes = plt.subplots(5, 1, figsize=(5, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    for i, ax in enumerate(axes):

        channel = mwi.channels_int[i]

        ax.plot(ds.frequency*1e-3,
                ds['sen2_interp'].isel(channel=i),
                marker='o', markersize=2, label='measured')

        ax.plot(ds.frequency*1e-3,
                ds['sen2_from_sen1'].isel(channel=i),
                marker='o', markersize=2, label='calculated from DSB')

    axes[0].legend()
    axes[-1].set_xlabel('Frequency [GHz]')
    axes[2].set_ylabel('Sensitivity')

    # logarithmic
    fig, axes = plt.subplots(5, 1, figsize=(5, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    for i, ax in enumerate(axes):

        channel = mwi.channels_int[i]

        ax.plot(ds.frequency*1e-3,
                ds['sen2_interp_dB'].isel(channel=i),
                marker='o', markersize=2, label='measured')

        ax.plot(ds.frequency*1e-3,
                ds['sen2_from_sen1_dB'].isel(channel=i),
                marker='o', markersize=2, label='calculated from DSB')

    axes[0].legend()
    axes[-1].set_xlabel('Frequency [GHz]')
    axes[2].set_ylabel('Sensitivity [dB]')

    #%% visualization: sensitivity graphs for bandpass region only
    fig, axes = plt.subplots(1, 5, sharey=True)

    for i, ax in enumerate(axes):

        ax.spines.top.set_visible(False)
        ax.spines.right.set_visible(False)

        ax.plot(ds.frequency*1e-3,
                ds.sen2_interp.isel(channel=i))

        ax.plot(ds.frequency*1e-3,
                ds.sen2_from_sen1.isel(channel=i))

        ax.set_xticks(np.arange(0, 300))
        dx = 1.25 - mwi.bandwidths[i]/2
        ax.set_xlim(mwi.freq_bw[i, 2]-dx, mwi.freq_bw[i, 3]+dx)

    # logarithmic
    fig, axes = plt.subplots(1, 5, sharey=True)

    for i, ax in enumerate(axes):

        ax.spines.top.set_visible(False)
        ax.spines.right.set_visible(False)

        ax.plot(ds.frequency*1e-3,
                ds.sen2_interp_dB.isel(channel=i))

        ax.plot(ds.frequency*1e-3,
                ds.sen2_from_sen1_dB.isel(channel=i))

        ax.set_xticks(np.arange(0, 300))
        dx = 1.25 - mwi.bandwidths[i]/2
        ax.set_xlim(mwi.freq_bw[i, 2]-dx, mwi.freq_bw[i, 3]+dx)

        ax.set_ylim(-10, 1)

    #%% visualization: sensitivity differences
    # difference between linear
    fig, axes = plt.subplots(5, 1, figsize=(5, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    for i, ax in enumerate(axes):

        channel = mwi.channels_int[i]

        ax.plot(ds.frequency*1e-3,
                ds.sen2_diff.isel(channel=i),
                marker='o', markersize=2)

    axes[-1].set_xlabel('Frequency [GHz]')
    axes[2].set_ylabel('Sensitivity: meas - calc(DSB)')

    # differences between dB
    fig, axes = plt.subplots(5, 1, figsize=(5, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    for i, ax in enumerate(axes):

        channel = mwi.channels_int[i]

        ax.plot(ds.frequency*1e-3,
                ds.sen2_diff_dB.isel(channel=i),
                marker='o', markersize=2)

    axes[-1].set_xlabel('Frequency [GHz]')
    axes[2].set_ylabel('Sensitivity: meas - calc(DSB) [dB]')

    #%% visualization: differences between dB for bandpass region with labels
    fig, axes = plt.subplots(1, 5, figsize=(6, 2.3), sharey=True,
                             constrained_layout=True, gridspec_kw=dict(
                                 width_ratios=mwi.bandwidths[::-1]))

    f = np.array([
        ds.frequency.where(
            (ds.frequency > mwi.freq_bw_MHz[i, 2]) & \
            (ds.frequency < mwi.freq_bw_MHz[i, 3]))
            for i in range(5)])
    ix_f = np.isnan(f).sum(axis=0) <= 3

    for j, (i, ax) in enumerate(zip(ds.channel.values, axes[::-1])):

        ax.spines.top.set_visible(False)
        ax.spines.right.set_visible(False)

        ax.axhline(y=0, color='k', linewidth=0.8)

        # frequencies in bandpass region
        f = dict(frequency=slice(mwi.freq_bw_MHz[j, 2], mwi.freq_bw_MHz[j, 3]))

        # plot all
        ax.plot(
            ds.frequency.sel(**f)*1e-3,
            ds.sen2_diff_dB.sel(channel=i, **f),
            color='darkgray',
            )

        # plot overlap in different color
        ax.plot(
            ds.frequency.sel(**f)*1e-3,
            ds.sen2_diff_dB.sel(channel=i).where(ix_f).sel(**f),
            color='dimgray',
            )

        # fit line
        a, b = ds.sen2_diff_dB.sel(channel=i, **f).polyfit(
            dim='frequency', deg=1).polyfit_coefficients
        ax.plot(
            ds.frequency.sel(**f)*1e-3,
            ds.frequency.sel(**f)*a + b,
            color='#577590', label='linear fit',
            )

        ax.set_xticks(np.arange(0, 300))
        ax.set_xlim(list(f.values())[0].start*1e-3,
                    list(f.values())[0].stop*1e-3)

        ax.set_yticks(np.arange(-10, 10, 0.2))
        ax.set_ylim(-.4, .65)

        ax.annotate(f'({abc[4-j]})', xy=(0.01, .99),
                    xycoords='axes fraction', ha='left', va='top')

        ax.annotate(mwi.freq_txt[j].split('\n')[0], xy=(0.5, 1),
                    xycoords='axes fraction', ha='center', va='bottom')

    axes[-1].legend(frameon=False, fontsize=7)

    axes[0].set_xlabel('Frequency [GHz]')
    axes[0].set_ylabel('Sensitivity difference [dB]')

    plt.savefig(os.path.join(
        os.environ['PATH_PLT'],
        'srf_diff.png'),
        dpi=300, bbox_inches='tight')

    #%% visualization: sensitivity differences normalized
    # difference between linear
    fig, axes = plt.subplots(5, 1, figsize=(5, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    for i, ax in enumerate(axes):

        channel = mwi.channels_int[i]

        ax.plot(ds.frequency*1e-3,
                ds.sen2_diff_norm.isel(channel=i),
                marker='o', markersize=2)

    axes[-1].set_xlabel('Frequency [GHz]')
    axes[2].set_ylabel('Sensitivity: meas - calc(DSB)')

    #%% visualization: scatter of differences
    # differences linear
    fig, axes = plt.subplots(5, 5, constrained_layout=True)

    for (i, j), ax in np.ndenumerate(axes):
        ax.scatter(
            ds.sen2_diff.isel(channel=i),
            ds.sen2_diff.isel(channel=j),
            s=2, c='k', alpha=0.5)

    # differences linear, where linear sensitivity is high enough
    fig, axes = plt.subplots(5, 5, constrained_layout=True)

    for (i, j), ax in np.ndenumerate(axes):

        ix = (ds.sen2_interp.isel(channel=i) > 0.001) & \
             (ds.sen2_interp.isel(channel=j) > 0.001)

        ax.scatter(
            ds.sen2_diff.isel(channel=i).sel(frequency=ix),
            ds.sen2_diff.isel(channel=j).sel(frequency=ix),
            s=2, c='k', alpha=0.5)

    # differences log
    fig, axes = plt.subplots(5, 5, constrained_layout=True)

    for (i, j), ax in np.ndenumerate(axes):
        ax.scatter(
            ds.sen2_diff_dB.isel(channel=i),
            ds.sen2_diff_dB.isel(channel=j),
            s=2, c='k', alpha=0.5)

    # normalized differences linear
    fig, axes = plt.subplots(5, 5, constrained_layout=True)

    for (i, j), ax in np.ndenumerate(axes):
        ax.scatter(
            ds.sen2_diff_norm.isel(channel=i),
            ds.sen2_diff_norm.isel(channel=j),
            s=2, c='k', alpha=0.5)

    #%% visualization: correlation
    fig, axes = plt.subplots(1, 3, figsize=(8, 4), constrained_layout=True)

    dfs = [
        sen2_diff_corr,
        sen2_diff_dB_corr,
        sen2_diff_norm,
        ]

    for i, ax in enumerate(axes):

        df = dfs[i]

        im = ax.pcolormesh(
            df.index,
            df.index,
            df,
            cmap='PuOr', shading='nearest', vmin=-1, vmax=1)

        for (i, j), z in np.ndenumerate(df.values):
            ax.annotate(text='{:0.2f}'.format(np.round(z,1)), xy=(14+j, 14+i),
                        xycoords='data', ha='center', va='center',
                        color='gray')

    fig.colorbar(im, ax=axes)

    #%%
    plt.close('all')