from django.contrib import messages
from django.core.mail import send_mail
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.shortcuts import redirect, render
from djeym.models import Placemark

from gotohome.settings import EMAIL_HOST_USER
from users.models import CustomUser

def map(request):
    if not request.user.is_authenticated or not request.user.is_shelter:
        messages.info(request,
                      'Только приюты могут пользоваться этой картой')
        return redirect('homepage:home')

    template_name = 'maps/shelter_maps.html'

    return render(request, template_name)

@receiver(post_save, sender=Placemark)
def notify_email(instance, **kwargs):
    if instance.is_user_marker and not instance.is_sended_admin_email:
        subject = 'Text subject'
        message = 'Text message - Url: ' + \
            'http(s)://your.domain/admin/djeym/placemark/{}/change/'.format(
                instance.pk
            )
        from_email = EMAIL_HOST_USER
        staff_users = list(
            i['email'] for i in CustomUser.objects.filter(
                is_staff=True
            ).values('email').distinct()
        )
        recipient_list = staff_users
        send_mail(subject, message, from_email,
                  recipient_list, fail_silently=False)

        instance.is_sended_admin_email = True
        instance.save()
    elif instance.active and instance.is_user_marker \
            and not instance.is_sended_user_email:
        subject = 'Text subject'
        message = 'Text message'
        from_email = EMAIL_HOST_USER
        recipient_list = [instance.user_email]
        send_mail(subject, message, from_email,
                  recipient_list, fail_silently=False)
        instance.is_sended_user_email = True
        instance.save()


from datetime import datetime

import sqlalchemy as sa

from .base import Model

class User(Model):
    __tablename__ = 'users'

    join_date = sa.Column(sa.DateTime, default=datetime.now())
    deep_link = sa.Column(sa.String(255))

    def __repr__(self) -> str:
        return self._repr(id=self.id,
                          join_date=self.join_date,
                          deep_link=self.deep_link)


#!/usr/bin/python3
"""print titles of hot top ten"""
import json
import requests
import sys

def top_ten(subreddit):
    """print hot top ten"""
    headers = {
        'User-Agent': 'Mozilla/5.0'
    }
    url = 'https://www.reddit.com/api/search_reddit_names.json'
    valid = requests.get(url, headers=headers, params={'query': subreddit})\
                    .json()
    if subreddit in valid['names']:
        top = requests.get('https://www.reddit.com/r/' + subreddit + '.json',
                           headers=headers).json()
        cnt = 0
        for dicti in top['data']['children']:
            if cnt < 10:
                print(dicti['data']['title'])
            cnt += 1
    else:
        print(None)


from typing import Tuple, Dict, List

import numpy as np
import pandas as pd
import torch

from BiLSTM.bilstm import get_all_masked
from DST.loader import Loader
from DataConstants import DataConstants
from STISE.module import improved_ses_all, improved_idw, ElmNet

def stise(eTag):
    dc = DataConstants(eTag)
    cities, indicator, stations = dc.getPublicData()

    for city in cities:
        loader = Loader(city, indicator, eTag)
        _, __, ___, ____, _____, ______, X_test_row_index, X_test_column_index = loader.train_test_loader(
            0.3, 128)

        masked_data = pd.read_csv(r'./data/' + eTag + '/' + city + '-' + indicator + '-masked.csv')
        extracted_data = pd.read_csv(r'./data/' + eTag + '/' + city + '-' + indicator + '-extracted.csv')
        # the furthest station is removed in the experiment 'e84'
        if eTag in ['e84', 'e86']:
            location_data = pd.read_csv(r'./data/' + city + "-station-location-selected.csv")
        else:
            location_data = pd.read_csv(r'./data/' + city + "-station-location.csv")

        # get indexes of training data and test data respectively
        X_test_row_index, X_test_column_index = \
            X_test_row_index.astype(int).tolist(), X_test_column_index.astype(int).tolist()
        X_test_index = [(r, c) for r, c in zip(X_test_row_index, X_test_column_index)]
        X_train_index = list(set(get_all_masked(masked_data)) - set(X_test_index))

        # Transform indexes into training sets and test sets
        """Obtain trainable or predictable samples according to the row and column index"""
        site_list = [p for p in masked_data.columns if 'A' in p]
        temp_result = improved_ses_all(masked_data, site_list=site_list)
        spat_result = improved_idw(raw_data=masked_data, sites_location=location_data, point_list=site_list)
        x_train, y_train = [], []
        for x_idx in X_train_index:
            row_index, col_index = x_idx
            if np.isnan(temp_result.iloc[row_index, col_index]) or np.isnan(spat_result.iloc[row_index, col_index]):
                continue
            x_train.append([temp_result.iloc[row_index, col_index], spat_result.iloc[row_index, col_index]])
            y_train.append(extracted_data.iloc[row_index, col_index])
        x_train, y_train = torch.tensor(x_train).to(torch.float), torch.tensor(y_train).to(torch.float)

        elm_net = ElmNet()
        elm_net.fine_ture(x_train, y_train)
        elm_net.eval()

        with torch.no_grad():
            for x_idx in X_test_index:
                row_index, col_index = x_idx
                x = torch.tensor([temp_result.iloc[row_index, col_index], spat_result.iloc[row_index, col_index]])
                y = float(elm_net.forward(x))
                masked_data.iloc[row_index, col_index] = y

        site_list = [p for p in masked_data.columns if 'A' in p]
        for i, r in masked_data.iterrows():
            nan_list, un_nan_list = [], []
            for site in site_list:
                if np.isnan(r[site]):
                    nan_list.append(site)
                else:
                    un_nan_list.append(site)
            if len(nan_list) == 0 or len(un_nan_list) == 0:
                continue
            new_r = r.copy(deep=True)
            new_r[nan_list] = float(new_r[un_nan_list].values.mean())
            masked_data.iloc[i, :] = new_r
        # masked_data = masked_data.interpolate(method='linear', limit_direction='both')

        for train_index in X_train_index:
            row_index, col_index = train_index
            masked_data.iloc[row_index, col_index] = np.nan

        infilled_file = "./results/" + eTag + '/' + city + "-" + indicator + "-stise-filling.csv"
        masked_data.to_csv(infilled_file, index=False)


from typing import Optional
import c4d
import json
import os

doc: c4d.documents.BaseDocument  # The active document
op: Optional[c4d.BaseObject]  # The active object, None if unselected

#exemple extraction initiale via og2ogr
#ogr2ogr -clipdst 2567613.1657820013 1108343.8926543007 2570413.1657820013 1111143.8926543007 "/Users/olivierdonze/switchdrive/Mandats/Parc_naturel_regional_vallee_Trient/c4d/maquettes_villages/Vernayaz/vernayaz_routes_TLM.geojson" "/Volumes/My Passport Pro/swisstopo/SWISSTLM3D_2022_LV95_LN02.gdb" "TLM_STRASSE"

CONTAINER_ORIGIN = 1026473

def main() -> None:
    fn = '/Users/olivierdonze/switchdrive/Mandats/Parc_naturel_regional_vallee_Trient/c4d/maquettes_villages/Vernayaz/vernayaz_routes_TLM.geojson'
    origin = doc[CONTAINER_ORIGIN]

    res = c4d.BaseObject(c4d.Onull)
    res.SetName(os.path.basename(fn))
    doc.InsertObject(res)
    with open(fn,encoding='utf-8') as f :
        data = json.loads(f.read())

        for feat in data['features']:
            objektart = feat['properties']['OBJEKTART']

            if feat['geometry']['type'] =='LineString':
                nb_seg = 1
                pts = [c4d.Vector(x,y,z) - origin for x,z,y in feat['geometry']['coordinates']]

            if feat['geometry']['type'] =='MultiLineString':
                coord = feat['geometry']['coordinates']
                nb_seg = len(coord)
                pts = []
                segs = []
                for i,points in enumerate(coord):
                    pts.extend([c4d.Vector(x,y,z) - origin for x,z,y in points])
                    segs.append(len(points))

            nb_pts = len(pts)
            sp = c4d.SplineObject(nb_pts,c4d.SPLINETYPE_LINEAR)
            sp.SetName(objektart)
            sp.SetAllPoints(pts)
            if nb_seg>1:
                sp.ResizeObject(nb_pts, nb_seg)
                for i,n in enumerate(segs):
                    sp.SetSegment(i,n,closed = False)

            sp.Message(c4d.MSG_UPDATE)
            sp.InsertUnderLast(res)

    c4d.EventAdd()

"""
def state():
    # Defines the state of the command in a menu. Similar to CommandData.GetState.
    return c4d.CMD_ENABLED
"""

if __name__ == '__main__':
    main()

swistopo/import geojson TLM3D_strasse after clip.py


import datetime
import requests
import json
import re
import os
import csv
from src.models.Uploads import Uploads
from src.models import Emails as Email
from src.views import Logger

BASE = os.getcwd()
LOGGER = Logger.init_logger(__name__)

def clean_filename(course_id: str, fname: str):
    """Cleans up given filename, removing undesired punctuation
    Args:
        course_id (str): course id
        fname (str): file name to clean up
    Returns:
        str: either cleaned filename or course_id if filename is over 160 chars long
    """
    LOGGER.debug(f"{course_id}, {fname}")
    handle_types(*[(course_id, ""), (fname, "")])

    file = re.sub(r"[^\w\s]", "", fname)
    filename = file.replace("recording", "").rstrip().replace(" ", "-")
    return course_id if len(filename) > 160 else filename

def create_dir_if_not_exists(path: str):
    """Creates directory if it doesn't exist already
    Args:
        path (str): path to directory
    Returns:
        str: path to directory
    """
    LOGGER.debug(f"{path}")
    handle_types(*[(path, "")])
    try:
        if not os.path.exists(path):
            os.makedirs(path)
    except Exception as e:
        msg = str(e)
        LOGGER.error(msg)
        raise e

    return path

def calculate_time(secs: int):
    """Converts given number of seconds into "%H:%M:%S" format
    Args:
        secs (int): number of seconds
    Returns:
        str: "%H:%M:%S" time format
    """
    LOGGER.debug(f"{secs}")
    handle_types(*[(secs, 0)])
    mins, secs = divmod(secs, 60)
    hours, mins = divmod(mins, 60)
    days, hours = divmod(hours, 24)
    time_in_session = datetime.time(hours, mins, secs)
    return time_in_session.strftime("%H:%M:%S")

def convert_date(date: str):
    """Converts given date to "%b %d,%Y" format. Example : Sep 03,2021
    Args:
        date (str): date
    Returns:
        str: date in "%b %d,%Y" format
    """
    LOGGER.debug(f"{date}")
    handle_types(*[(date, "")])
    try:
        date_obj = datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ")
    except ValueError:
        msg = "Incorrect data format, should be %Y-%m-%dT%H:%M:%S.%fZ"
        LOGGER.error(msg)
        raise ValueError(msg)
    return date_obj.strftime("%b %d,%Y")

def set_search_start_date(weeks: int):
    """Returns date {weeks} from now
    Args:
        weeks (int): number of weeks to go back
    Returns:
        str: date in "%Y-%m-%d" format
    """
    LOGGER.debug(f"Setting search from {weeks} weeks ago")
    handle_types(*[(weeks, 0)])

    start_date = datetime.datetime.now() - datetime.timedelta(weeks=int(weeks))
    return start_date.strftime("%Y-%m-%d")

def get_headers(auth_or_bearer):
    """Helper function - template for returning headers to make API calls depending on auth or bearer being required
    Args:
        auth_or_bearer (str): String variable for header
    Returns:
        [dict]: Header for making requests
    """
    LOGGER.debug(f"{auth_or_bearer}")
    handle_types(*[(auth_or_bearer, "")])

    return {
        "Authorization": auth_or_bearer,
        "Content-Type": "application/json",
        "Accept": "application/json",
    }

def clean_chat_message(message):
    """Removes unwanted whitespace and characters from Collab chat entries prior to saving
    Args:
        message (str): Chat message on given recording
    Returns:
        [str]: Cleaned message
    """
    LOGGER.debug(f"{message}")
    handle_types(*[(message, "")])
    clean = re.compile("<.*?>")
    return re.sub(clean, "", message)

def save_chatfile_as_csv(fname, json_info, header):
    file = open(fname, "w", encoding="utf-8")
    writer = csv.writer(file)
    writer.writerow(header)
    for json_row in json_info:
        writer.writerow(
            [
                json_row["userName"],
                clean_chat_message(json_row["body"]),
                json_row["relativeEventTime"],
                json_row["id"],
            ]
        )
    file.close()

def list_entry(results, size=None, recording_type=0):
    """Generates download report entry data depending on recording found or not found
    Returns:
        [dict]: Dict of data for entry to Collab download report
    """
    LOGGER.debug(f"{results}, {size}, {recording_type}")
    handle_types(*[(results, {}), (recording_type, 0)])

    if not type(size) is int and size != None:
        msg = f"Recording size should be type(int) and not None"
        LOGGER.error(msg)
        raise TypeError(msg)

    try:
        if recording_type == 1:
            list_data = {
                "recording_id": results["id"],
                "recording_name": results["name"],
                "duration": results["duration"],
                "storage_size": size,
                "created": results["created"],
            }
        else:
            list_data = {
                "recording_id": results["id"],
                "recording_name": results["name"],
                "msg": 403,
            }
    except KeyError as ke:
        raise KeyError(f"Expected key in dict: {ke}")

    return list_data

def recording_storage_size(url: str):
    """returns content length for given recording url
    Args:
        url (str): recording url
    Returns:
        int: size of content length
    """
    LOGGER.debug(f"{url}")
    handle_types(*[(url, "")])

    try:
        r = requests.get(url, stream=True, headers={"Accept-Encoding": None})
    except requests.exceptions.MissingSchema as e:
        LOGGER.error(str(e))
        raise requests.exceptions.MissingSchema(str(e))

    return int(r.headers.get("content-length", 0))

def return_filename(course_id, rec, filetype, chat=""):
    LOGGER.debug(f"{course_id}, {rec}, {filetype}, {chat}")
    handle_types(*[(course_id, ""), (rec, {}), (filetype, ""), (chat, "")])

    try:
        return (
            f"Collab-2-Panopto-",
            f"{chat}"
            f"{course_id}-{rec['recording_id']}_"
            f"{clean_filename(course_id, rec['recording_name'])}"
            f"{filetype}"
        )

    except KeyError as ke:
        raise KeyError(f"Expected key in dict: {ke}")

def get_courseId_folderId_pairs(uuid_id_pairs):
    """Takes dict of uuid:course_id mappings from Collab data, uses course_id to search
        for corresponding Panopto folder_id
    Args:
        uuid_id_pairs ([dict]): uuid to course_id mappings
    Returns:
        [dict]: Course_id to Panopto folder_id mappings
    """
    LOGGER.debug(f"{uuid_id_pairs}")
    handle_types(*[(uuid_id_pairs, {})])

    uploader = Uploads()
    courseId_folderId_pairs = check_json_exists()

    missing_folders_list = []
    for courseId in uuid_id_pairs.values():
        if courseId in courseId_folderId_pairs.keys():
            continue
        else:
            folderId = uploader.get_folder_id_from_course_id(courseId)
            if folderId is not None:
                courseId_folderId_pairs[courseId] = folderId
            else:
                missing_folders_list.append(courseId)

    save_as_json(courseId_folderId_pairs, f"{BASE}/data")

    # message = Email.create_email()
    # email = Email.attach_missing_folders_csv(message, missing_folders_list)
    # Email.send_info_email(email)

    return courseId_folderId_pairs, missing_folders_list

def save_as_json(courseId_folderId_pairs, json_dir_path):
    """Saves or updates courseId_folderId_pairs as json to /data folder
    Args:
        courseId_folderId_pairs (dict): Dict of course_id to Panopto folder_id mappings
        json_dir_path (path): Path to /data folder
    """
    LOGGER.debug(f"{courseId_folderId_pairs}, {json_dir_path}")
    handle_types(*[(courseId_folderId_pairs, {}), (json_dir_path, "")])
    dir = create_dir_if_not_exists(json_dir_path)
    with open(f"{dir}/courseId_folderId_pairs.json", "w") as fp:
        json.dump(courseId_folderId_pairs, fp, indent=4)

    return courseId_folderId_pairs

def get_downloads_list(downloads_path):
    """Gets list of mp4 files in Collab /downloads folder
    Args:
        downloads_path (path): Path to Collab downloads folder
    Returns:
        [list]: List of paths to each mp4 download file
    """
    LOGGER.debug(f"{downloads_path}")
    handle_types(*[(downloads_path, "")])

    downloads_list = []
    for file in os.scandir(downloads_path):
        if file.path.endswith(".mp4"):
            downloads_list.append(file.path)

    return downloads_list

def get_folder_download_matches(courseIds_folderIds, downloads_list):
    """ Matches course_id strings from mp4 files in Collab /downloads to Panopto folder_id
        Adds {folder_id: [download]} to matches dict if not exists, else appends download
    Args:
        courseIds_folderIds (dict): Dict of course_id:folder_id mappings
        downloads_list (list(mp4)): List of mp4 recordings in Collab /downloads
    Returns:
        [dict]: Mapping of course downloads to corresponding Panopto folder_ids
    """
    LOGGER.debug(f"{courseIds_folderIds}, {downloads_list}")
    handle_types(*[(courseIds_folderIds, {}), (downloads_list, [])])

    matches = {}
    for download in downloads_list:
        for course_id, folder_id in courseIds_folderIds.items():
            if course_id in download:
                if folder_id in matches:
                    matches[folder_id].append(download)
                else:
                    matches[folder_id] = [download]
    return matches

def logger_msg(*vars):
    return [str(type(var)) for var in vars]

def handle_response(r):
    """Generic function for API response handling and error logging.
    Args:
        r (res): Response from API call
    Returns:
        Parsed json or None, depending on status_code
    """
    if r.status_code == 200:
        res = json.loads(r.text)
        return res
    elif r.status_code == 403:
        LOGGER.warning(f"Status code 403: (might be private recording!) - {r}")
        return None
    elif r.status_code == 404:
        LOGGER.warning(f"Status code 404: Not Found - {str(r)}")
        return None
    else:
        LOGGER.warning(f"Unknown response for: {str(r)}")
        return None

def handle_types(*received_expected):
    """Generic helper function for handling TypeErrors
    Args:
        rec_exp (list(tuples)): Unpackable list of (rec, exp) tuples of arbitrary length
    Raises:
        TypeError: if type(rec) != type(exp)
    """
    for pair in received_expected:
        if not type(pair[0]) is type(pair[1]):
            msg = f"Type {type(pair[1])} expected, received: {type(pair[0])}"
            LOGGER.error(msg)
            raise TypeError(msg)

def handle_res_content(key, json):
    if not key in json:
        raise KeyError(f"Expected key {key} in response json")

def check_json_exists():
    """Loads courseId_folderId_pairs.json as dict if exists, returns empty dict if not
    Returns:
        [dict]: dict(courseId_folderId_pairs)
    """
    try:
        with open(f"{BASE}/data/courseId_folderId_pairs.json") as f:
            courseId_folderId_pairs = json.load(f)
    except:
        courseId_folderId_pairs = {}

    return courseId_folderId_pairs

def return_absent_present(present_on_ppto, list_of_recordings):
    absent, present = [], []

    # Turn list of dicts into a single dict
    dict_present_recordings = {}
    for item in present_on_ppto:
        dict_present_recordings[list(item.keys())[0]] = list(item.values())[0]

    # Check if recording is already present on panopto
    for rec in list_of_recordings:
        rec_short_name = get_new_name(os.path.basename(rec))

        # Check for both the short name and the full name.
        # Full name can be on panopto when we are checking before the recording has been renamed
        if not rec_short_name in list(
            dict_present_recordings.keys()
        ) and not os.path.basename(rec) in list(dict_present_recordings.keys()):
            # Neither were found, the recording is absent on panopto
            absent.append(rec)
        else:
            # Present with short name
            if rec_short_name in dict_present_recordings:
                present.append(
                    {rec_short_name: dict_present_recordings[rec_short_name]}
                )
            # Present with full name
            else:
                present.append(
                    {
                        os.path.basename(rec): dict_present_recordings[
                            os.path.basename(rec)
                        ]
                    }
                )

    return absent, present

def check_upload_results(failed_uploads, recordings_not_uploaded):
    if len(recordings_not_uploaded) > 0:
        LOGGER.warning(
            f"Some recordings weren't uploaded properly : {recordings_not_uploaded }"
        )
    if failed_uploads != recordings_not_uploaded:
        LOGGER.warning(
            f"failed_uploads and recordings_not_uploaded should be identical - they aren't!!!"
        )

def get_date_created(path, rec_ids_date_created):
    for ele in rec_ids_date_created:
        for id, date in ele.items():
            if id in path:
                return date.rsplit("T", 1)

def get_new_name(old_name):
    return old_name.rsplit("_", 1)[1]


import cv2

#import numpy as np

#img=np.zeros((512,512))
img=cv2.imread('/home/avani/Desktop/dogs.jpg')

#cutimg=img[0:200,0:300]

cv2.line(img,(0,0),(20,30),(0,0,255),3)
cv2.circle(img,(50,50),24,(0,255,0),2)
cv2.putText(img,"heya",(50,60),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0),4)
cv2.imshow("cute dog",img)
#cv2.imshow("cut",cutimg)
cv2.waitKey(0)
cv2.destroyAllWindows()


from datetime import datetime
from elasticsearch import Elasticsearch as ES, RequestsHttpConnection
from requests_aws4auth import AWS4Auth

class ESSearch:
    def __init__(self, connection, es_index='covid_tweets'):
        aws_auth = AWS4Auth(connection['ACCESS_KEY'], connection['SECRET_KEY'], 'us-east-2', 'es')
        self.es = ES(
            hosts=[{'host': connection['AWS_HOST'], 'port': 443}],
            http_auth=aws_auth,
            use_ssl=True,
            verify_certs=True,
            connection_class=RequestsHttpConnection,
            timeout=200
        )
        self.es_index = es_index

    def format_query(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=None):
        if len(keywords) == 0:
            keywords=None
        queries = []
        time_range = {}
        if startDateString:
            startDate = datetime.strptime(startDateString, '%m/%d/%Y')
            time_range['gte'] = startDate.strftime('00-%d-%m-%Y')
        if endDateString:
            endDate = datetime.strptime(endDateString, '%m/%d/%Y')
            time_range['lte'] = endDate.strftime('23-%d-%m-%Y')
        if time_range:
            queries.append({
                'range': {
                    'date': time_range
                }
            })
        if keywords:
            queries.append({
                "query_string": {
                    "query": keywords,
                    "fields" : ["text"],
                    "default_operator" : "or"
                }
            })
        if user:
            queries.append({
                "match": {
                    "user_name" : user
                }
                })
        if tweettype:
            queries.append({
                "terms": {
                    "tweet_type" : tweettype,
                }
            })
        print(queries)
        return queries

    def get_doc(self, tweet_id):
        retval = self.es.get(index=self.es_index, id=tweet_id, doc_type='document')
        return retval

    def get_user_tweet(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=None):
        queries = self.format_query(keywords, startDateString=startDateString, endDateString=endDateString, tweettype=tweettype, user=user)
        total_qry = { "size" : 250, 'query' : {'bool': {'must' : queries
        }}}
        retval = self.es.search(index=self.es_index, doc_type='document', body=total_qry)
        return retval

    def count(self, keywords, startDateString=None, endDateString=None, tweettype=None, user=False):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        total_qry = {}
        retval = {}
        if user:
            total_qry = {"size" : 0,
                        'query': {'bool': {'must': queries }},
                        "aggs" : {
                            "users_count" : {
                                "cardinality" : {
                                    "field" : "user_name"
                                }
                            }
                        }}
            retval = self.es.search(index=self.es_index, doc_type='document', body=total_qry)
        else:
            total_qry = { 'query' : {'bool': {'must' : queries}}}
            retval = self.es.count(index=self.es_index, body=total_qry)
        return retval

    def agg_qry(self, keywords, startDateString=None, endDateString=None, tweettype=None):
        return retval

    def query(self, keywords, startDateString=None, endDateString=None, size=None, tweettype=None):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        if size:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': size, 'query': {'bool': {'must': queries }}})
            for tw in retval['hits']['hits']:
                yield tw
        else:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': 1000, 'query': {'bool': {'must': queries }}})
            total = retval["hits"]["total"]
            print(total)
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            while scroll_size > 0:
                print("Scrolling...")
                for tw in retval['hits']['hits']:
                    yield tw
                if not size:
                    retval = self.es.scroll(scroll_id=sid, scroll='1m')
                    sid = retval['_scroll_id']
                    scroll_size = len(retval['hits']['hits'])
            self.es.clear_scroll(scroll_id=sid)

    def sizequery(self, keywords, startDateString=None, endDateString=None, size=None, tweettype=None, random=True):
        queries = self.format_query(keywords, startDateString, endDateString, tweettype)
        queries = {'bool': {'must': queries }}
        print(queries)
        if random:
            queries = {"function_score": {
                "query": queries,
                "random_score": {},
            }}
        if size < 1000:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': size, 'query': queries})
            for tw in retval['hits']['hits']:
                yield tw
        else:
            retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
                body={'size': 1000, 'query': queries})
            total = retval["hits"]["total"]
            so_far = len(retval['hits']['hits'])
            if size == None:
                size = total
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            while scroll_size > 0 and so_far <= size:
                print(so_far)
                print(size)
                print(total)
                print("Scrolling...")
                for tw in retval['hits']['hits']:
                    yield tw
                retval = self.es.scroll(scroll_id=sid, scroll='1m')
                sid = retval['_scroll_id']
                scroll_size = len(retval['hits']['hits'])
                so_far = so_far + scroll_size
            self.es.clear_scroll(scroll_id=sid)
        """count = 0
        retval = self.es.search(index=self.es_index, scroll='1m', doc_type='document',
            body={'size': 1000, 'query': {'bool': {'must': queries }}})
        total = retval["hits"]["total"]
        print(total)
        sid = retval['_scroll_id']
        scroll_size = len(retval['hits']['hits'])
        #count += scroll_size
        while count < size:
            print("Scrolling...")
            for tw in retval['hits']['hits']:
                yield tw

            retval = self.es.scroll(scroll_id=sid, scroll='1m')
            sid = retval['_scroll_id']
            scroll_size = len(retval['hits']['hits'])
            count += scroll_size
        self.es.clear_scroll(scroll_id=sid)"""

if __name__ == "__main__":
    from test_credentials import AWS_PROFILE
    import json
    e = ESSearch(AWS_PROFILE)
    count=0
    future_of_work_string="employee employees employed employing employ jobless job jobs work working works worked unemployable unemployed wfh"
    test_string = '"self care"'
    test_string = '"covid"'
    retarr = []
    #print(e.count(test_string, tweettype=["original", "quote", "reply"]))
    for doc in e.sizequery(test_string, tweettype=["original"], size=10000):
        #print(doc["_source"])
        retarr.append(doc["_source"])
        count+=1
    f = open("lindsay_tweets.json", "w")

    json.dump(retarr, f)
    print(count)


import asyncio
from itertools import combinations

from poke_env.player.player import Player
from poke_env.player.random_player import RandomPlayer

from src.pokemon.bot.SimpleRuleBasedPlayer import RuleBasedPlayer

EVAL_EPISODES = 2

async def main():
    print("Evaluating Bots!")

    concurrent = 1

    player_list = [
        RuleBasedPlayer(battle_format="gen8randombattle", max_concurrent_battles=concurrent, save_replays=True),
        # SimpleRuleBasedPlayer(battle_format="gen8randombattle", max_concurrent_battles=concurrent),
        # MaxDamagePlayer(battle_format="gen8randombattle", max_concurrent_battles=concurrent),
        RandomPlayer(battle_format="gen8randombattle", max_concurrent_battles=concurrent),
    ]

    # Making the bots play against each other
    for battle in combinations(player_list, 2):
        p1: Player = battle[0]
        p2: Player = battle[1]
        p1_name = p1.username.split()[0]
        p2_name = p2.username.split()[0]

        p1.reset_battles()
        p2.reset_battles()

        await p1.battle_against(p2, n_battles=EVAL_EPISODES)
        print(f"{p1_name} ({p1.n_won_battles} / {p2.n_won_battles}) {p2_name}")

if __name__ == "__main__":
    asyncio.get_event_loop().run_until_complete(main())


import numpy as np
from copy import deepcopy
from src.search.mcts.worker import MCTSWorker
from src.search.search import ParallelSearchAlgorithm
from torch.multiprocessing import Pipe

class MCTS(ParallelSearchAlgorithm):
    """ Monte Carlo Tree Search, with root parallelization."""

    def __init__(self, config):
        super().__init__(config)

        # Create workers
        pipes = [Pipe() for _ in range(self.num_workers)]
        self.channels = [p[0] for p in pipes]
        self.workers = [MCTSWorker(config, pipes[i][1]) for i in range(self.num_workers)]
        for w in self.workers:
            w.start()

    def search(self, state, iters=None):
        iters = iters if iters is not None else self.num_iters
        for c in self.channels:
            c.send({
                "command": "search",
                "state": deepcopy(state),
                "iters": iters,
            })
        msg = [c.recv() for c in self.channels]
        pi = np.mean([m["pi"] for m in msg], axis=0)
        qvals = np.mean([m["q"] for m in msg], axis=0)
        vals = np.mean([m["v"] for m in msg], axis=0)
        return {
            "pi": pi,
            "q": qvals,
            "v": vals,
        }


from typing import List
import pytest
from ledger import TransactionLog, Transaction, AccountNotFoundException

def test_create_account_with_500_balance():
    transaction_log = TransactionLog()
    transaction_log.create_account('john wick', 500.)

    assert transaction_log.get_account_balance('john wick') == 500.

def test_create_account_with_zero_balance():
    transaction_log = TransactionLog()
    transaction_log.create_account('james wick')

    assert transaction_log.get_account_balance('james wick') == 0.

def test_add_transaction():
    transaction_log = TransactionLog()
    transaction_log.create_account('john')
    transaction_log.create_account('james')
    tx = Transaction(tx_date = '2021-5-24 11:00', src_acct = 'john', dst_acct = 'james', tx_value = 100.)
    transaction_log.add_transaction(tx)

    assert transaction_log.get_account_balance('james') == 100
    assert transaction_log.get_account_balance('john') == -100

def test_get_balance_at_date():
    transaction_log = TransactionLog()
    transaction_log.create_account('john')
    transaction_log.create_account('james')

    transaction_log.add_transaction(Transaction(src_acct='john', dst_acct='james', tx_date = '2021-1-10 08:00', tx_value = 100))
    transaction_log.add_transaction(Transaction(src_acct='john', dst_acct='james', tx_date = '2021-2-10 08:00', tx_value = 100))
    transaction_log.add_transaction(Transaction(src_acct='john', dst_acct='james', tx_date = '2021-3-10 08:00', tx_value = 100))

    assert transaction_log.get_account_balance_at('james', '2021-2-25 12:00') == 200.

def test_invalid_account_raises_account_not_found():
    transaction_log = TransactionLog()
    with pytest.raises(AccountNotFoundException):
        transaction_log.get_account_balance('hendrix')

def test_get_transactions_for_account():
    tx1 = Transaction(src_acct='john', dst_acct='james', tx_date = '2021-1-10 08:00', tx_value = 100)
    tx2 = Transaction(src_acct='peter', dst_acct='sarah', tx_date = '2021-2-10 08:00', tx_value = 50)
    transaction_log = TransactionLog([tx1, tx2])

    assert transaction_log.get_transactions('peter') == [tx2]

def test_get_transactions_for_account_to_date():
    tx1 = Transaction(src_acct='john', dst_acct='james', tx_date = '2021-1-1 08:00', tx_value = 50)
    tx2 = Transaction(src_acct='peter', dst_acct='sarah', tx_date = '2021-1-2 08:00', tx_value = 50)
    tx3 = Transaction(src_acct='john', dst_acct='james', tx_date = '2021-1-3 08:00', tx_value = 50)
    tx4 = Transaction(src_acct='peter', dst_acct='sarah', tx_date = '2021-1-4 08:00', tx_value = 50)
    transaction_log = TransactionLog([tx1, tx2, tx3, tx4])

    assert transaction_log.get_transactions_to_date('james', '2021-1-2 23:00') == [tx1]


from itertools import zip_longest
import logging

import numpy as np

from hyperspy.misc.axis_tools import check_axes_calibration
from hyperspy.misc.array_tools import are_aligned

_logger = logging.getLogger(__name__)

def _get_shapes(am, ignore_axis):
    if ignore_axis is not None:
        try:
            ignore_axis = am[ignore_axis]
        except ValueError:
            pass
    sigsh = tuple(axis.size if (ignore_axis is None or axis is not
                                ignore_axis)
                  else 1 for axis in am.signal_axes) if am.signal_dimension != 0 else ()

    navsh = tuple(axis.size if (ignore_axis is None or axis is not
                                ignore_axis)
                  else 1 for axis in am.navigation_axes) if am.navigation_dimension != 0 else ()
    return sigsh, navsh

def are_signals_aligned(*args, ignore_axis=None):
    if len(args) < 2:
        raise ValueError(
            "This function requires at least two signal instances"
        )

    args = list(args)
    am = args.pop().axes_manager

    sigsh, navsh = _get_shapes(am, ignore_axis)
    while args:
        amo = args.pop().axes_manager
        sigsho, navsho = _get_shapes(amo, ignore_axis)

        if not (are_aligned(sigsh[::-1], sigsho[::-1]) and
                are_aligned(navsh[::-1], navsho[::-1])):
            return False
    return True

def _check_and_get_longest_axis(axes):
    """Return the longest axis from a list of axes.

    In the case of ties, the first element in the list
    will be returned. Logs a warning if the axes have
    different calibrations.
    """
    only_left = filter(lambda x: x is not None, axes)
    longest_idx, longest = sorted(
        enumerate(only_left),
        key=lambda x: x[1].size,
        reverse=True,
    )[0]

    # Exit early if not DataAxis objects - None is used
    # in broadcast_signals() below as a filler so we
    # skip it here.
    def _check_wrapper(ax1, ax2):
        if ax1 is None or ax2 is None:
            return True

        return check_axes_calibration(ax1, ax2)

    # Returns a list of bools, where False
    # indicates an axis with a different
    # calibration to the longest axis
    axes_check = [
        _check_wrapper(axes[longest_idx], axes[i])
        for i in range(len(axes))
        if i != longest_idx
    ]

    if not all(axes_check):
        _logger.warning(
            "Axis calibration mismatch detected along axis "
            f"{longest.index_in_axes_manager}. The "
            f"calibration of signal {longest_idx} along "
            "this axis will be applied to all signals "
            "after stacking."
        )

    return longest

def broadcast_signals(*args, ignore_axis=None):
    """Broadcasts signals according to the HyperSpy broadcasting rules.

    signal and navigation spaces are each separately broadcasted according to
    the numpy broadcasting rules. One axis can be ignored and left untouched
    (or set to be size 1) across all signals.

    Parameters
    ----------
    *args : BaseSignal
        Signals to broadcast together
    ignore_axis : {None, str, int, Axis}
        The axis to be ignored when broadcasting

    Returns
    -------
    list of signals

    """
    from hyperspy.signal import BaseSignal

    if len(args) < 2:
        raise ValueError(
            "This function requires at least two signal instances"
            )
    if any([not isinstance(a, BaseSignal) for a in args]):
        raise ValueError("Arguments must be of signal type.")
    args = list(args)
    if not are_signals_aligned(*args, ignore_axis=ignore_axis):
        raise ValueError("The signals cannot be broadcasted")
    else:
        if ignore_axis is not None:
            for s in args:
                try:
                    ignore_axis = s.axes_manager[ignore_axis]
                    break
                except ValueError:
                    pass
        new_nav_axes = []
        new_nav_shapes = []
        for axes in zip_longest(
            *[s.axes_manager.navigation_axes for s in args], fillvalue=None
        ):
            longest = _check_and_get_longest_axis(axes)
            new_nav_axes.append(longest)
            new_nav_shapes.append(
                longest.size
                if (ignore_axis is None or ignore_axis not in axes)
                else None
            )
        new_sig_axes = []
        new_sig_shapes = []
        for axes in zip_longest(
            *[s.axes_manager.signal_axes for s in args], fillvalue=None
        ):
            longest = _check_and_get_longest_axis(axes)
            new_sig_axes.append(longest)
            new_sig_shapes.append(
                longest.size
                if (ignore_axis is None or ignore_axis not in axes)
                else None
            )

        results = []
        new_axes = new_nav_axes[::-1] + new_sig_axes[::-1]
        new_data_shape = new_nav_shapes[::-1] + new_sig_shapes[::-1]
        for s in args:
            data = s._data_aligned_with_axes
            sam = s.axes_manager
            sdim_diff = len(new_sig_axes) - len(sam.signal_axes)
            while sdim_diff > 0:
                slices = (slice(None),) * sam.navigation_dimension
                slices += (None, Ellipsis)
                data = data[slices]
                sdim_diff -= 1
            thisshape = new_data_shape.copy()
            if ignore_axis is not None:
                _id = new_data_shape.index(None)
                newlen = data.shape[_id] if len(data.shape) > _id else 1
                thisshape[_id] = newlen
            thisshape = tuple(thisshape)
            if data.shape != thisshape:
                data = np.broadcast_to(data, thisshape)

            ns = s._deepcopy_with_new_data(data)
            ns.axes_manager._axes = [ax.copy() for ax in new_axes]
            ns.get_dimensions_from_data()
            results.append(ns.transpose(signal_axes=len(new_sig_axes)))
        return results


from typing import Tuple, Union

class TrieNode:
    """
    A class to represent a node in a trie
    that contains only the pointers to the used characters
    (as opposed to the whole vocabulary array).

    ...

    Attributes
    ----------

    children : dict[str, TrieNode]
        pointers to the children nodes (for successive characters)
    """

    def __init__(self) -> None:
        self.children = {}

class Trie:

    def __init__(self, terminal: str = '$'):
        self.root = TrieNode()
        self.terminal = terminal

    def insert(self, word: str) -> None:
        """Inserts a word into the trie.

        Args:
            word (str): word to insert
        """
        if self.terminal in word:
            raise Exception(f'Explicit terminal ({self.terminal}) use is not allowed within words!')

        node = self.root
        for char in word:
            if char not in node.children:
                node.children[char] = TrieNode()

            node = node.children[char]

        pointer_dict = node.children

        # proper terminating the word
        if self.terminal not in pointer_dict:
            pointer_dict[self.terminal] = None

    def search_prefix(self, prefix: str) -> Tuple[bool, Union[TrieNode, None]]:
        """Searches for given prefix in the trie

        Args:
            prefix (str): search term

        Returns:
            bool: True, if the prefix exists in trie
            TrieNode: - THE node that contains all suffixes of the prefix in trie,
                           if the prefix is contained
                      - None, if the prefix is not found
        """
        if self.terminal in prefix:
            raise Exception(f'Explicit terminal ({self.terminal}) use is not allowed within words!')

        node = self.root
        for char in prefix:
            if char in node.children:
                node = node.children[char]
            else:
                return False, None

        return True, node

    def search(self, word: str) -> bool:
        found, node = self.search_prefix(word)

        if not node:
            return False

        if self.terminal not in node.children:
            return False

        return True

test_trie1 = Trie()

vocabulary = ['cikla', 'cifar', 'aminokiselina', 'aminosulfat', 'amino']
[test_trie1.insert(word) for word in vocabulary]

search_terms = ['ami', 'amino', 'cikla', 'zirafa', 'cifara']
search_results = []
prefix_results = []
for term in search_terms:
    search_results.append(test_trie1.search(term))
    prefix_results.append(test_trie1.search_prefix(term)[0])

# test_trie1.insert('ami$') # causes exception
# test_trie1.search('ami$') # causes exception


"""
Train on Emotion dataset
"""
import os
import time
import numpy as np
import torch
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import Utils
import math
import Config
from tqdm import trange

def emotrain(model, data_loader, args, focus_emo):
	"""
	:data_loader input the whole field
	"""
	# start time
	time_st = time.time()
	decay_rate = args.decay

	# Load in the training set and validation set
	train_loader = data_loader['train']
	dev_loader = data_loader['dev']

	labels = train_loader['label']
	audio_feats = train_loader['audio']
	raw_audio_feats = train_loader['raw_audio']

	# Optimizer
	lr = args.lr
	model_opt = optim.Adam(model.parameters(), lr=lr)

	print("Dataset : {} \n Emotion rates {}".format(args.dataset,Config.data_count))

	# Raise the .train() flag before training
	model.train()

	over_fitting = 0
	cur_best = -1e10
	cur_best_loss = 100
	glob_steps = 0
	report_loss = 0
	for epoch in range(1, args.epochs + 1):
		model_opt.param_groups[0]['lr'] *= decay_rate
		labels, audio_feats, raw_audio_feats = Utils.shuffle_lists(labels, audio_feats, raw_audio_feats)
		print("===========Epoch {}==============".format(epoch))
		print("-{}-{}".format(epoch, Utils.timeSince(time_st)))
		for bz in trange(len(labels)):
			# Tensorize a dialogue, a dialogue is a batch

			label = Utils.ToTensor(labels[bz])
			audio_feat = torch.from_numpy(np.array(audio_feats[bz])).float()
			raw_audio_feat = torch.from_numpy(np.array(raw_audio_feats[bz])).float()

			audio_len = raw_audio_feats[bz][:,:,0]
			mask_len = torch.from_numpy(np.array(audio_len)).float()
			audio_lens = Utils.ToAudioLens(audio_len)

			label = Variable(label)
			audio_feat = Variable(audio_feat)
			raw_audio_feat = Variable(raw_audio_feat)

			if args.gpu != None:
				os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
				device = torch.device("cuda: 0")
				model.cuda(device)
				label = label.cuda(device)
				audio_feat = audio_feat.cuda(device)
				raw_audio_feat = raw_audio_feat.cuda(device)
				mask_len = mask_len.cuda(device)

			log_prob = model(raw_audio_feat, audio_lens, mask_len)
			loss = comput_class_loss(log_prob, label)
			loss.backward()
			report_loss += loss.item()
			glob_steps += 1

			# gradient clip
			torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5)

			model_opt.step()
			model_opt.zero_grad()

			if glob_steps % args.report_loss == 0:
				print("Steps: {} Loss: {} LR: {}".format(glob_steps, report_loss/args.report_loss, model_opt.param_groups[0]['lr']))
				report_loss = 0

		# validate
		pAccs, y_true, y_pred = emoeval(model=model,
										data_loader=dev_loader,
										args=args,
										focus_emo=focus_emo)
		print("Evaluation Metric [{}, {}, {}, {}, {}, {}]".format('happy', 'anger', 'sad', 'neutral', 'WAcc', 'UWAcc'))
		print("Validate: ACCs-WA-UWA {}".format(pAccs[:-1]))

		last_best = pAccs[-1]  # loss
		if last_best < cur_best_loss:
			Utils.model_saver(model, args.save_dir, args.dataset)
			cur_best_loss = last_best
			over_fitting = 0
		else:
			over_fitting += 1
		if over_fitting >= args.patience:
			break

def comput_class_loss(log_prob, target):
	""" Weighted loss function """
	loss = F.nll_loss(log_prob, target.view(target.size(0)), reduction='sum')
	loss /= target.size(0)

	return loss

def emoeval(model, data_loader, args, focus_emo):
	""" data_loader only input 'dev' """
	model.eval()

	TP = np.zeros([args.class_num], dtype=np.long) # recall
	TP_FN = np.zeros([args.class_num], dtype=np.long) # gold
	focus_idx = [Config.label_index[emo] for emo in focus_emo]

	audio_feats = data_loader['audio']
	labels = data_loader['label']
	raw_audio_feats = data_loader['raw_audio']

	val_loss = 0
	y_true=[]
	y_pred=[]

	for bz in range(len(labels)):
		label = Utils.ToTensor(labels[bz])
		audio_feat = torch.from_numpy(np.array(audio_feats[bz])).float()
		raw_audio_feat = torch.from_numpy(np.array(raw_audio_feats[bz])).float()

		audio_len = raw_audio_feats[bz][:,:,0]
		mask_len = torch.from_numpy(np.array(audio_len)).float()
		audio_lens = Utils.ToAudioLens(audio_len)

		label = Variable(label)
		audio_feat = Variable(audio_feat)
		raw_audio_feat = Variable(raw_audio_feat)

		if args.gpu != None:
			os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
			device = torch.device("cuda: 0")
			model.cuda(device)
			label = label.cuda(device)
			audio_feat = audio_feat.cuda(device)
			raw_audio_feat = raw_audio_feat.cuda(device)
			mask_len = mask_len.cuda(device)

		log_prob = model(raw_audio_feat, audio_lens, mask_len)

		# print(log_prob, label)
		# val loss
		loss = comput_class_loss(log_prob, label)
		val_loss += loss.item()

		# accuracy
		emo_predidx = torch.argmax(log_prob, dim=1)
		emo_true = label.view(label.size(0))

		a = emo_true.cpu().tolist()
		for i in a:
			y_true.append(i)
		b = emo_predidx.cpu().tolist()
		for j in b:
			y_pred.append(j)

		for lb in range(emo_true.size(0)):
			idx = emo_true[lb].item()
			TP_FN[idx] += 1
			if idx in focus_idx:
				if emo_true[lb] == emo_predidx[lb]:
					TP[idx] += 1

	f_TP = [TP[Config.label_index[w]] for w in focus_emo]
	f_TP_FN = [TP_FN[Config.label_index[w]] for w in focus_emo]
	Recall = [np.round(tp/tp_fn*100, 2) if tp_fn>0 else 0 for tp,tp_fn in zip(f_TP,f_TP_FN)]
	wRecall = sum([r * w / sum(f_TP_FN) for r,w in zip(Recall, f_TP_FN)])
	uRecall = sum(Recall) / len(Recall)

	# Accuracy of each class w.r.t. the focus_emo, the weighted acc, and the unweighted acc
	Total = Recall + [np.round(wRecall,2), np.round(uRecall,2)] + [np.round(val_loss,3)]

	# Return to .train() state after validation
	model.train()

	return Total, y_true, y_pred


import os
import errno
import fs
from fs.walk import Walker
from fnmatch import fnmatch
from fs.mountfs import MountFS, MountError
from fs.multifs import MultiFS
from fs.path import abspath, forcedir, normpath
from psslib.contentmatcher import ContentMatcher
from psslib.utils import istextfile
from psslib.driver import _build_match_context_dict, LINE_MATCH, LINE_CONTEXT
from typing import Any, List, Optional, Union, Tuple
from flask import send_file, stream_with_context, Response
from airflow_code_editor.utils import read_mount_points_config, get_plugin_config, get_plugin_int_config

__all__ = [
    "RootFS",
]

STAT_FIELDS = [
    "st_mode",
    "st_ino",
    "st_dev",
    "st_nlink",
    "st_uid",
    "st_gid",
    "st_size",
    "st_atime",
    "st_mtime",
    "st_ctime",
]

SEND_FILE_CHUNK_SIZE = 8192

def split(pathname: str):
    "Split a pathname, returns tuple (head, tail)"
    pathname = pathname.rstrip("/")
    i = pathname.rfind("/") + 1
    if i == 0:
        return ("/", pathname)
    else:
        return pathname[: i - 1], pathname[i:]

class RootFS(MountFS):
    "Root filesystem with mountpoints"

    def __init__(self):
        super().__init__()
        mounts = read_mount_points_config()
        # Set default fs (root)
        self.default_fs = MultiFS()
        self.tmp_fs = fs.open_fs("mem://")
        self.default_fs.add_fs("tmp", self.tmp_fs, write=False, priority=0)
        self.root_fs = [fs.open_fs(v.path) for v in mounts.values() if v.default][0]
        self.default_fs.add_fs("root", self.root_fs, write=True, priority=1)
        # Mount other fs
        for k, v in mounts.items():
            if not v.default:
                self.mount("/~" + k, fs.open_fs(v.path))

    def mount(self, path, fs_):
        "Mounts a host FS object on a given path"
        if isinstance(fs_, str):
            fs_ = fs.open_fs(fs_)
        path_ = forcedir(abspath(normpath(path)))
        for mount_path, _ in self.mounts:
            if path_.startswith(mount_path):
                raise MountError("mount point overlaps existing mount")
        self.mounts.append((path_, fs_))
        # Create mountpoint on the temporary filesystem
        self.tmp_fs.makedirs(path_, recreate=True)

    def path(self, *parts: List[str]):
        "Return a FSPath instance for the given path"
        return FSPath(*parts, root_fs=self)

    def find_files(
        self,
        path: str = "/",
        filter: Union[List[str], str, None] = None,
        exclude: Optional[List[str]] = None,
        max_depth: Optional[int] = None,
    ):
        "Walk a filesystem, yielding FSPAth"
        if exclude == []:
            exclude = None
        if isinstance(filter, str):
            filter = [filter]
        walker = Walker(
            ignore_errors=True,
            filter=filter,
            exclude=exclude,
            exclude_dirs=exclude,
            max_depth=max_depth,
        )
        fs = self.root_fs
        for filename in walker.files(fs=fs, path=path):
            yield self.path(filename)

    def search(
        self,
        query: str,
        search_context: Optional[int] = None,
        path: str = "/",
        filter: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        max_depth: Optional[int] = None,
    ):
        "Search for pattern in files"
        if search_context is None:
            search_context = get_plugin_int_config("search_context")
        if exclude is None:
            exclude = get_plugin_config("ignored_entries").split(",")

        result = []
        matcher = ContentMatcher(pattern=query.encode("utf-8"))
        for path in self.find_files(filter=filter, exclude=exclude, max_depth=max_depth):
            try:
                with path.open("rb") as f:
                    if istextfile(f):
                        f.seek(0)
                        matches = list(matcher.match_file(f))
                        if not search_context:
                            for match in matches:
                                context = match[0].decode("utf-8")
                                row_number = match[1]
                                result.append(
                                    {
                                        "row_number": row_number,  # matching row number
                                        "context_first_row": row_number,  # context first row number
                                        "context": context,  # context (matching row)
                                        "path": path.path,  # file path
                                    }
                                )
                        else:
                            for row, context, context_first_row in prepare_search_context(f, matches, search_context):
                                result.append(
                                    {
                                        "row_number": row,  # matching row number
                                        "context_first_row": context_first_row,  # context first row number
                                        "context": context,  # context
                                        "path": path.path,  # file path
                                    }
                                )
            except (OSError, IOError):
                pass

        return result

def prepare_search_context(f, matches, search_context) -> Tuple[int, str, int]:
    f.seek(0)
    match_context_dict = _build_match_context_dict(matches, search_context, search_context)
    lines = []
    context_first_row = 0
    row_number = 0

    prev_was_blank = False
    had_context = False
    for current_row_number, current_line in enumerate(f, 1):
        current_line = current_line.decode("utf-8")
        result, match = match_context_dict.get(current_row_number, (None, None))
        if result is None:
            prev_was_blank = True
            continue

        elif result == LINE_MATCH:
            row_number = current_row_number
            if current_row_number < context_first_row or context_first_row == 0:
                context_first_row = current_row_number
            lines.append(current_line)

        elif result == LINE_CONTEXT:
            if prev_was_blank and had_context:
                context = "".join(lines)
                yield row_number, context, context_first_row
                lines = []
                context_first_row = current_row_number
            if context_first_row < context_first_row or context_first_row == 0:
                context_first_row = current_row_number
            lines.append(current_line)
            had_context = True
        prev_was_blank = False

    if lines:
        context = "".join(lines)
        yield row_number, context, context_first_row

class FSPath(object):
    def __init__(self, *parts: List[str], root_fs: RootFS) -> None:
        self.root_fs = root_fs
        self.path = os.path.join("/", *parts)

    def open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
        "Open the file pointed by this path and return a file object"
        return self.root_fs.open(
            self.path,
            mode=mode,
            buffering=buffering,
            encoding=encoding,
            errors=errors,
            newline=newline,
        )

    @property
    def name(self) -> str:
        "The final path component"
        return split(self.path)[1]

    @property
    def parent(self):
        "The logical parent of the path"
        return self.root_fs.path(split(self.path)[0])

    def touch(self, mode=0o666, exist_ok=True):
        "Create this file"
        return self.root_fs.touch(self.path)

    def rmdir(self) -> None:
        "Remove this directory"
        self.root_fs.removedir(self.path)

    def unlink(self, missing_ok: bool = False) -> None:
        "Remove this file"
        try:
            self.root_fs.remove(self.path)
        except fs.errors.ResourceNotFound:
            if not missing_ok:
                raise FileNotFoundError(self.path)

    def delete(self) -> None:
        "Remove this file or directory"
        if self.is_dir():
            self.rmdir()
        else:
            self.unlink()

    def stat(self):
        "File stat"
        info = self.root_fs.getinfo(self.path, namespaces=["stat"])
        if not info.has_namespace("stat"):
            return os.stat_result([None for _ in STAT_FIELDS])
        return os.stat_result([info.raw["stat"].get(field) for field in STAT_FIELDS])

    def is_dir(self) -> bool:
        "Return True if this path is a directory"
        try:
            return self.root_fs.isdir(self.path)
        except Exception:
            return False

    def resolve(self):
        "Make the path absolute"
        return self.root_fs.path(os.path.realpath(self.path))

    def exists(self):
        "Check if this path exists"
        return self.root_fs.exists(self.path)

    def iterdir(self, show_ignored_entries=False):
        "Iterate over the files in this directory"
        ignored_entries = get_plugin_config("ignored_entries").split(",")
        mount_points = [x[0].rstrip("/") for x in self.root_fs.mounts]
        for name in sorted(self.root_fs.listdir(self.path)):
            skip = False
            if not show_ignored_entries:
                fullpath = os.path.join(self.path, name)
                # Skip mount points
                if fullpath in mount_points:
                    skip = True
                # Ship hidden files
                for patter in ignored_entries:
                    if fnmatch(fullpath if patter.startswith("/") else name, patter.strip()):
                        skip = True
            if not skip:
                yield self.root_fs.path(self.path, name)

    def size(self) -> Optional[int]:
        "Return file size for files and number of files for directories"
        try:
            if self.is_dir():
                return len(self.root_fs.listdir(self.path))
            else:
                return self.root_fs.getsize(self.path)
        except fs.errors.FSError:
            return None

    def move(self, target) -> None:
        "Move/rename a file or directory"
        target = self.root_fs.path(target)
        if target.is_dir():
            self.root_fs.move(self.path, (target / self.name).path)
        else:
            self.root_fs.move(self.path, target.path)

    def read_file_chunks(self, chunk_size: int = SEND_FILE_CHUNK_SIZE):
        "Read file in chunks"
        with self.root_fs.openbin(self.path) as f:
            while True:
                buffer = f.read(chunk_size)
                if buffer:
                    yield buffer
                else:
                    break

    def send_file(self, as_attachment: bool):
        "Send the contents of a file to the client"
        if not self.exists():
            raise FileNotFoundError(errno.ENOENT, "File not found", self.path)
        elif self.root_fs.hassyspath(self.path):
            # Local filesystem
            if as_attachment:
                # Send file as attachment (set Content-Disposition: attachment header)
                try:
                    # flask >= 2.0 -  download_name replaces the attachment_filename
                    return send_file(
                        self.root_fs.getsyspath(self.path),
                        as_attachment=True,
                        download_name=self.name,
                    )
                except TypeError:
                    return send_file(
                        self.root_fs.getsyspath(self.path),
                        as_attachment=True,
                    )
            else:
                return send_file(self.root_fs.getsyspath(self.path))
        else:
            # Other filesystems
            response = Response(stream_with_context(self.read_file_chunks()))
            if as_attachment:
                content_disposition = "attachment;filename={}".format(self.name)
                response.headers["Content-Disposition"] = content_disposition
            return response

    def write_file(self, data: Union[str, bytes], is_text: bool) -> None:
        "Write data to a file"
        self.root_fs.makedirs(self.parent.path, recreate=True)
        if is_text:
            self.root_fs.writetext(self.path, data)
        else:
            self.root_fs.writebytes(self.path, data)

    def read_text(self, encoding=None, errors=None) -> str:
        "Get the contents of a file as a string"
        return self.root_fs.readtext(self.path, encoding=encoding, errors=errors)

    def read_bytes(self) -> bytes:
        "Get the contents of a file as bytes"
        return self.root_fs.readbytes(self.path)

    def __str__(self) -> str:
        return self.path

    def __repr__(self) -> str:
        return self.path

    def __truediv__(self, key):
        try:
            path = os.path.join(self.path, key)
            return self.root_fs.path(path)
        except TypeError:
            return NotImplemented

    def __eq__(self, other) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path == other.path

    def __hash__(self) -> int:
        try:
            return self._hash
        except AttributeError:
            self._hash = hash(self.path)
            return self._hash

    def __lt__(self, other: Any):
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path < other.path

    def __le__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path <= other.path

    def __gt__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path > other.path

    def __ge__(self, other: Any) -> bool:
        if not isinstance(other, FSPath):
            return NotImplemented
        return self.path >= other.path


# functions for deciphering text

import re
import bisect
import collections
import copy

import numpy as np
import scipy

from wordbank import Token, Wordbank

vocab = None # modern dictionary word -> ID
inv_vocab = None # modern dictionary. ID -> word

def basic_substitute(ciphertext, wordbank):
    '''
    replace strings in ciphertext with known decryptions from wordbank

    ciphertext: string
    wordbank: instance of Wordbank
    '''

    token_list = tokenize_ciphertext(ciphertext)
    return ' '.join(token.plaintext for token in token_list)

# def apply_wordbank(token_list, wordbank):
#     for token in token_list:
#         if token in wordbank:
#             token.plaintext = wordbank[token]

# def load_vocab(filename):
#     '''
#     load a vocabulary.

#     input should be a sorted list
#     '''
#     _vocab = {} # word -> index
#     _inv_vocab = [] # index -> word
#     with open(filename) as f:
#         for idx, line in enumerate(f):
#             _vocab = {line: idx}
#             _inv_vocab.append(line)

#     _vocab['<unk>'] = len(_vocab)
#     _inv_vocab.append('<unk>')

#     return _vocab, _inv_vocab

# noisy channel model
def lm(token_list):
    '''
    read a context and return one-hot predictions over the output space

    output:
    '''

    # PLACEHOLDER UNTIL I GET A REAL LM
    # fake language model that simply uses the length. shorter words are better.

    score = 0

    # probability of each word. these are not history dependent
    likelihood = np.array([1 / len(word) for word in vocab.words])
    logprob = np.log(likelihood / likelihood.sum())

    # get score over history
    for token in token_list:
        score += logprob(vocab.lookup_word(token.plaintext))

    # add history score to score for each possible addition
    return score + logprob

def prior(token, wordbank):
    '''
    unigram log-probability of each token given its approximate location. returns an array of the same shape as lm
    model P(decoding|encoding) with a beta distribution (because it's normalized 0 to 1)
    '''

    left, right = wordbank.query_range(token)

    # find boundary words in modern vocab
    modern_left = bisect.bisect_left(modern_dict, left.plaintext)
    modern_right = bisect.bisect(modern_dict, right.plaintext)

    # set parameters of the beta distribution with mean where the word is relative to its anchor points
    mean = (int(token) - int(left)) / (int(right) - int(left))
    variance = 0.1
    n = mean * (1 - mean) / variance

    dist_params = {
        a: mean * n,
        b: (1 - mean) * n,
        loc: modern_left, # shift and scale to be about [0,1), and have variance 1
        scale: modern_right - modern_left,
    }
    if dist_params['a'] < 1 or dist_params['b'] < 1:
        raise ValueError('a, b must be > 1 to have a unimodal distribution')

    # probability of each word is the integral of the PDF from
    result = np.zeros(10000,)
    for word_idx in range(modern_left, modern_right):
        result[vocab[word]] = scipy.stats.beta.logcdf(word_idx + 1, **dist_params) - scipy.stats.beta.logcdf(word_idx, **dist_params)
    return result

Candidate = collections.namedtuple('Candidate', ['logprob', 'tokens'])

def decode(token_list, wordbank, beam_size=16):
    '''beam search decoding'''

    beam = [Candidate(logprob=0, tokens=[])]

    for token in token_list:
        new_candidates = []

        for beam_idx, candidate in enumerate(beam):
            if token.ciphertype is None or token.plaintext != '<unk>':
                # probably a hard-coded string. doesn't affect our model's probability
                candidate.tokens.append(token)
                new_candidates.append(candidate)

            elif token.ciphertype == 'dict':
                # prior is a beta distribution over the bounds established by the wordbank table

                lm_logprob = lm(candidate.tokens)
                prior_logprob = prior(wordbank, tokens)

                logprob = lm_logprob + prior_logprob

            elif token.ciphertype == 'table':
                # prior is a uniform distribution over all english words
                logprob = lm(candidate.tokens)

            # update beam
            if token.ciphertype == 'dict' or token.ciphertype == 'table':
                # get k best extensions to the current sequence and add them to the beam
                k = (beam_size - beam_idx)
                best_extensions = np.argpartition(logprob, len(vocab) - np.arange(k))[-k:] # np array of ints, shape: (k,)

                for ext_idx in best_extensions:
                    new_token = copy.copy(token)
                    new_token.plaintext = inv_vocab[ext_idx]
                    new_candidates.append(Candidate(logprob, candidate.tokens + [new_token]))

        beam = sorted(new_candidates)[-beam_size:] # todo: could save log(beam) time here by doing a partial sort only
        print(beam)

    print(beam)
    return beam[-1]

if __name__ == '__main__':
    wordbank = Wordbank()
    wordbank.load('wordbanks/wordbank.without.2880')

    # vocab, inv_vocab = load_vocab('dict.modern')
    ciphertext = '[556]^ 586[26]- Ferdinand 2 [678]^ 95 ? [1235]^ y ? [433]^ [79]^ [664]^'
    tokenized_ct = tokenize_ciphertext(ciphertext)

    # One hour baseline
    print(tokenized_ct)

    # Beam search experiments

    # print(decode(tokenize_ciphertext(ciphertext), wordbank))


import os
import yaml
import numpy as np
import open3d as o3d
import random
from datasets.scannet200.owis_splits import KNOWN_CLASSES_LABELS,UNKNOWN_CLASSES_LABELS
from datasets.scannet200.scannet200_constants import CLASS_LABELS_200,VALID_CLASS_IDS_200
from copy import copy
# OUTPUT_2 = "output-scans/predictions-ours-2classes/"
label2id = {l:id for id,l in zip(VALID_CLASS_IDS_200, CLASS_LABELS_200)}
def generate_color_palette(x):
    palette = []

    while len(palette) < x:
        r, g, b = random.sample(range(256), 3)
        color = (r/255.0, g/255.0, b/255.0)
        if color[2] > 0:
            continue
        palette.append(color)

    return palette

def read_ply_file(file_path):
    # Read the ply file
    point_cloud = o3d.io.read_triangle_mesh(file_path)
    return point_cloud

model_name = "experiment_name"

for split in ['A','B','C']:
    for task in ['task1','task2','task3']:
        PATH = f"eval_output/{split}/{task}/instance_evaluation_{model_name}_0/decoder_-1/"
        label_database = "data/processed/scannet200/label_database.yaml"
        OUTPUT = f"output-scans/{model_name}/{split}/{task}/"
        OUTPUT_2 = f"output-scans/gt/{split}/{task}/"
        OUTPUT_3 = f"output-scans/input/{split}/{task}/"
        with open(label_database) as f:
            x = yaml.safe_load(f)

        # read label database
        cls_ids = x.keys()
        # from dict keys to list
        cls_ids = [int(i) for i in cls_ids]
        for file in os.listdir(PATH):
            pc = []
            pco = []
            SCENE = file.split(".")[0]
            # if SCENE in good_scenes:
            PATH_PLY = f"data/raw/scannet_test_segments/scans/{SCENE}/{SCENE}_vh_clean_2.labels.ply"
            PATH_PLY_1 = f"data/raw/scannet_test_segments/scans/{SCENE}/{SCENE}_vh_clean_2.ply"

            # read label database
            try:
                label = f"data/processed/scannet200/instance_gt/validation/{SCENE}.txt"
                # read label file
                with open(label) as f:
                    lines = f.readlines()
                unique_ids = np.unique(np.asarray(lines))
                unique_ids = [int(i[:-1]) for i in unique_ids]
                classes = [i//1000 for i in unique_ids if i !=0]
                # if task != 'task3':
                if UNKNOWN_CLASSES_LABELS[split][task] != None:
                    UNKNOWNS =  [label2id[i] for i in UNKNOWN_CLASSES_LABELS[split][task]]
                else:
                    UNKNOWNS = []

                instances = []
                if not os.path.exists(OUTPUT):
                    os.makedirs(OUTPUT)
                if not os.path.exists(OUTPUT_2):
                    os.makedirs(OUTPUT_2)
                if not os.path.exists(OUTPUT_3):
                    os.makedirs(OUTPUT_3)

                with open(f"{PATH}{SCENE}.txt") as f:
                    instances = f.readlines()
                # sort instances by probablity instances.split(" ")[2]
                instances = sorted(instances, key=lambda x: float(x.split(" ")[2]), reverse=False)

                pc = read_ply_file(PATH_PLY)
                pco = read_ply_file(PATH_PLY_1)
                points = np.asarray(pc.vertices)
                indices = []
                for instance in instances:
                    path = instance.split(" ")[0].strip()
                    cls = instance.split(" ")[1].strip()
                    # if(int(cls) not in cls_ids):
                    #     continue

                    with open(os.path.join(PATH, path), "r") as f:
                        mask = f.readlines()
                    for idx, point in enumerate(mask):
                        if(point == "1\n"):
                            indices.append(idx)
                            # change color of point based on its instance id
                            if(int(cls) == 0 or int(cls) == 1 or int(cls) == 3):
                                pc.vertex_colors[idx] = [0.5, 0.5, 0.5]
                            elif(int(cls) == 3000):
                                pc.vertex_colors[idx] = [31/255,119/255,180/255]
                            else:
                                pc.vertex_colors[idx] = [42/255,157/255,37/255]
                    indices = np.unique(np.asarray(indices))
                        # indices that are not in the list
                    left_indices = np.setdiff1d(np.arange(len(mask)), indices)
                    for index in left_indices:
                        pc.vertex_colors[index] = [0.5, 0.5, 0.5]
                    # save new point cloud
                    o3d.io.write_triangle_mesh(os.path.join(OUTPUT, SCENE + "-3d_owis.ply"), pc)

                    pc = read_ply_file(PATH_PLY)
                    points = np.asarray(pc.vertices)
                    for i in range(points.shape[0]):
                    # change color of point based on its instance id
                        if(int(lines[i][:-1])//1000 == 0 or int(lines[i][:-1])//1000 == 3 or int(lines[i][:-1])//1000 == 1): # make gray if 0 (nothing) or 3 (floor) or 1 (wall)
                            pc.vertex_colors[i] = [0.5, 0.5, 0.5]
                        elif(int(lines[i][:-1])//1000 in UNKNOWNS):
                            pc.vertex_colors[i] = [31/255,119/255,180/255]#[23/255,124/255,172/255]
                        else:
                            pc.vertex_colors[i] = [42/255,157/255,37/255] # green
                    o3d.io.write_triangle_mesh(os.path.join(OUTPUT_2, SCENE + "-gt.ply"), pc)
                    o3d.io.write_triangle_mesh(os.path.join(OUTPUT_3, SCENE + "-input.ply"), pco)
            except:
                pass


'''Some helper functions for PyTorch, including:
    - get_mean_and_std: calculate the mean and std value of dataset.
    - msr_init: net parameter initialization.
    - progress_bar: progress bar mimic xlua.progress.
'''
import os
import sys
import time
import math

import torch.nn as nn
import torch.nn.init as init

import numpy as np

def get_mean_and_std(dataset):
    '''Compute the mean and std value of dataset.'''
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2)
    mean = torch.zeros(3)
    std = torch.zeros(3)
    print('==> Computing mean and std..')
    for inputs, targets in dataloader:
        for i in range(3):
            mean[i] += inputs[:,i,:,:].mean()
            std[i] += inputs[:,i,:,:].std()
    mean.div_(len(dataset))
    std.div_(len(dataset))
    return mean, std

def init_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            init.kaiming_normal(m.weight, mode='fan_out')
            if m.bias:
                init.constant(m.bias, 0)
        elif isinstance(m, nn.BatchNorm2d):
            init.constant(m.weight, 1)
            init.constant(m.bias, 0)
        elif isinstance(m, nn.Linear):
            init.normal(m.weight, std=1e-3)
            if m.bias:
                init.constant(m.bias, 0)

def computedistance2points(point1, point2):
    return math.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)

def load_explore_traj(imgpath):
    if os.path.isdir(imgpath):
        imgs = []
        locations = []
        for name in os.listdir(imgpath):
            if name.endswith('.png'):
                imgs.append(name)
        imgs = sorted(imgs)
        for i in range(2,len(imgs),5):
            img  = imgs[i]
            img = img.split('.')[0]
            coord_x = float(img.split('_')[2])
            coord_y = float(img.split('_')[3])
            locations.append([coord_x, coord_y])
        return locations
    else:
        print("Not valid path")
        raise IOError

def load_navigation_traj_goal(path):
    position = []
    target = []
    with open(path,'r') as f:
        for cow in f.readlines():
            coord = list(map(float, cow.strip().split()))
            if len(target) == 0:
                target = [coord[2], coord[3]]
            position.append([coord[0], coord[1]])
    return position[:-1], target

_, term_width = os.popen('stty size', 'r').read().split()
term_width = int(term_width)

TOTAL_BAR_LENGTH = 65.
last_time = time.time()
begin_time = last_time
def progress_bar(current, total, msg=None):
    global last_time, begin_time
    if current == 0:
        begin_time = time.time()  # Reset for new bar.

    cur_len = int(TOTAL_BAR_LENGTH*current/total)
    rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1

    sys.stdout.write(' [')
    for i in range(cur_len):
        sys.stdout.write('=')
    sys.stdout.write('>')
    for i in range(rest_len):
        sys.stdout.write('.')
    sys.stdout.write(']')

    cur_time = time.time()
    step_time = cur_time - last_time
    last_time = cur_time
    tot_time = cur_time - begin_time

    L = []
    L.append('  Step: %s' % format_time(step_time))
    L.append(' | Tot: %s' % format_time(tot_time))
    if msg:
        L.append(' | ' + msg)

    msg = ''.join(L)
    sys.stdout.write(msg)
    for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3):
        sys.stdout.write(' ')

    # Go back to the center of the bar.
    for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2):
        sys.stdout.write('\b')
    sys.stdout.write(' %d/%d ' % (current+1, total))

    if current < total-1:
        sys.stdout.write('\r')
    else:
        sys.stdout.write('\n')
    sys.stdout.flush()

def format_time(seconds):
    days = int(seconds / 3600/24)
    seconds = seconds - days*3600*24
    hours = int(seconds / 3600)
    seconds = seconds - hours*3600
    minutes = int(seconds / 60)
    seconds = seconds - minutes*60
    secondsf = int(seconds)
    seconds = seconds - secondsf
    millis = int(seconds*1000)

    f = ''
    i = 1
    if days > 0:
        f += str(days) + 'D'
        i += 1
    if hours > 0 and i <= 2:
        f += str(hours) + 'h'
        i += 1
    if minutes > 0 and i <= 2:
        f += str(minutes) + 'm'
        i += 1
    if secondsf > 0 and i <= 2:
        f += str(secondsf) + 's'
        i += 1
    if millis > 0 and i <= 2:
        f += str(millis) + 'ms'
        i += 1
    if f == '':
        f = '0ms'
    return f

def adj2edges(adj):
    m = adj.shape[0]
    edges = []
    for i in range(m):
        for j in range(i):
            if adj[i, j] == 1:
                edges.append([i,j])
    return edges

def cos_sim(vector_a, vector_b):
    """
    Compute the cosine similarity
    :param vector_a: vector a
    :param vector_b: vector b
    :return: sim
    """
    vector_a = np.mat(vector_a)
    vector_b = np.mat(vector_b)
    num = float(vector_a * vector_b.T)
    denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)
    cos = num / denom
    sim = 0.5 + 0.5 * cos
    return sim


#!/usr/bin/env python3
from guizero import App, Combo, PushButton, Text
from screeninfo import get_monitors
from python_mpv_jsonipc import MPV
from threading import Thread
import socket
import subprocess
import os
import json
import time
import numpy as np
os.environ["DISPLAY"] = ":0.0"

sdobSocketKiller = 1
def thread_sdobSocket():
    global sdobSocketKiller
    global csock
    global dataSet
    print('Start Thread Socket')
    while sdobSocketKiller:
        try:
            (bytes, address) = csock.recvfrom(8196)
            msg = bytes.decode('utf-8')
            print('address:', address, 'recv', msg)
            # msgJson = json.loads(msg)
            # if type(msgJson) is dict:
            #    for k in msgJson.keys():
            #        setattr(dataSet, k, msgJson[k])
            #        print(k, msgJson[k])
            #else:
            #    setattr(dataSet, "socketdata", msg)
        except:
            time.sleep(0.2)
    # print('Stop Thread Socket')

def thread_getFrames(file):
    global dataSet
    global statusText
    statusText.value = "Getting KeyFrame Locations"
    cmd = "ffprobe -loglevel error -select_streams v:0 -show_entries packet=pts_time,flags -of csv=print_section=0 \"{}\" | awk -F',' '/K/ {{print $1}}'".format(file)
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    frameList = []
    for line in proc.stdout.readlines():
        frameList.append(line.decode('utf-8').rstrip())
    setattr(dataSet, "frameList", frameList)
    statusText.value = "Keyframes Found, Finished Looking"

def thread_processVideo():
    global dataSet
    global statusText

    videoRoot = '/tmp'
    videoFiles = []
    videoExt = '.mp4'
    videoDest = '/home/pi/Videos/120.mp4'

    if (hasattr(dataSet, "fileext")):
        videoExt = getattr(dataSet, "fileext")

    if (hasattr(dataSet, "slate")):
        # Slate
        sTime = float(getattr(dataSet, "slate")) - 2.0
        if (sTime < 0):
            sTime = 0.00
        eTime = float(getattr(dataSet, "slate")) + 5.0
        sTimes = splitVideoFileKeyTimes(sTime, eTime)
        # print("Slate Start Adding", sTime - sTimes[0])
        # print("Slate End Adding", sTimes[1] - eTime)
        videoFileName = 'sdobSlate{}'.format(videoExt)
        videoFile = '{}/{}'.format(videoRoot, videoFileName)
        videoFiles.append(videoFileName)
        splitVideoFile(getattr(dataSet, "filename"), sTimes[0], sTimes[1], videoFile)

    if (hasattr(dataSet, "exit")):
        statusText.value = "Splitting Video Skydive"
        # Skydive
        sTime = float(getattr(dataSet, "exit")) - 2.0
        if (sTime < 0):
            sTime = 0.00
        eTime = float(getattr(dataSet, "exit")) + 40.0
        sTimes = splitVideoFileKeyTimes(sTime, eTime)
        # print("Skydive Start Adding", sTime - sTimes[0])
        # print("Skydive End Adding", sTimes[1] - eTime)
        videoFileName = 'sdobExit{}'.format(videoExt)
        videoFile = '{}/{}'.format(videoRoot, videoFileName)
        videoFiles.append(videoFileName)
        splitVideoFile(getattr(dataSet, "filename"), sTimes[0], sTimes[1], videoFile)

    statusText.value = "Combining Video Slate and Skydive"
    # Recombine
    combineVideoFiles(videoRoot, videoFiles, videoDest)
    statusText.value = "Done Processing Video"

def globalClicked():
    if (hasattr(dataSet, "playing") and getattr(dataSet, "playing") == "1"):
        mpv_player.stop()

def selectButton(t):
    if (hasattr(dataSet, "playing") and getattr(dataSet, "playing") == "1"):
        return

    if (hasattr(dataSet, "team") and hasattr(dataSet, "round") and getattr(dataSet, "round") == t + 1):
        rX = getattr(r, "round_{}".format(str(t)))
        rX.bg = "blue"
        setattr(r, "round_{}".format(str(t)), rX)
        setattr(dataSet, "round", t + 1)
        print('Playing {} {}'.format(getattr(dataSet, "team"), getattr(dataSet, "round")))
    else:
        rKeys = r.__dict__.keys()
        for k in r.__dict__.keys():
            rX = getattr(r, k)
            rX.bg = "white"
            setattr(r, k, rX)

        rX = getattr(r, "round_{}".format(str(t)))
        rX.bg = "orange"
        setattr(r, "round_{}".format(str(t)), rX)
        setattr(dataSet, "round", t + 1)

def teamChanged(t):
    setattr(dataSet, "team", t)

def submitButton(t, r):
    # print("Submitting", getattr(dataSet, "filename"), getattr(dataSet, "slate"), getattr(dataSet, "exit"))
    if (hasattr(dataSet, "playing") and getattr(dataSet, "playing") == "1"):
        return

    if (hasattr(dataSet, "filename") and hasattr(dataSet, "team") and hasattr(dataSet, "round")):
        print('Uploading For {:s} Round: {:d}'.format(getattr(dataSet, "team"), getattr(dataSet, "round")))

    if (hasattr(dataSet, "filename") and hasattr(dataSet, "slate") and hasattr(dataSet, "exit")):
        print('Splitting Video At', getattr(dataSet, "slate"), 'and', getattr(dataSet, "exit"));

        # Start to Stop
        # splitVideoFile(getattr(dataSet, "filename"), getattr(dataSet, "slate"), getattr(dataSet, "exit"), "/tmp/test1.mp4")
        statusText.value = "Processing Video"
        processVideoThread = Thread(target = thread_processVideo)
        processVideoThread.setDaemon(True)
        processVideoThread.start()

def combineVideoFiles(rootFolder, videoArr, outputFilename):
    videoListPath = rootFolder + "/sdobVideoList.txt"
    f = open(videoListPath, "w")
    for v in videoArr:
        f.write("file '{}'\n".format(v))
    f.close()
    vidCmd = 'ffmpeg -y -f concat -i  "{}" -c copy {}'.format(videoListPath, outputFilename)
    os.system(vidCmd)
    # proc = subprocess.Popen(vidCmd, shell=True, stderr=subprocess.PIPE)
    # for line in proc.stderr.readlines():
    #     statusText.value = line.decode('utf-8').rstrip()

def splitVideoFileKeyTimes(startingTime, endingTime):
    ret = np.zeros(2, dtype = float)
    startTime = 0.00
    endTime = 0.00
    checkStart = float(startingTime)
    checkEnd = float(endingTime)
    # print(startingTime, endingTime)
    for f in getattr(dataSet, "frameList"):
        # print('Keyframe', f)
        if (float(f) <= checkStart):
            ret[0] = float(f)
        if (endTime == 0.00 and float(f) >= checkEnd):
            ret[1] = float(f)
        if (ret[0] != 0.00 and ret[1] != 0.00):
            break
    return ret

def splitVideoFile(filename, startTime, endTime, outputFilename):
    extraArgs = ""
    if (endTime != 0.00):
        extraArgs = extraArgs + " -t " + str(endTime - startTime)

    # print("StartKey", startTime, "EndKey", endTime)
    vid1Cmd = 'ffmpeg -y -i "{}" -ss {} {} -c copy {}'.format(filename, startTime, extraArgs, outputFilename)
    os.system(vid1Cmd)
    statusText.value = "Done!"
    # print(vid1Cmd)
    # proc = subprocess.Popen(vid1Cmd, shell=True, stderr=subprocess.PIPE)
    # for line in proc.stderr.readlines():
    #     statusText.value = line.decode('utf-8').rstrip()

def playnewVideo():
    mpv_player.loop = False
    mpv_player.play("/home/pi/Videos/120.mp4")
    csock.sendto(str.encode('{"event":"playing"}'), ssock_file)

def openButton():
    if (hasattr(dataSet, "playing") and getattr(dataSet, "playing") == "1"):
        return

    filename = app.select_file(folder="/media/", filetypes=[["Media Files", "*.mp4 *.mov *.vob *.wmv *.mpg *.mpeg *.mkv *.m4v *.avi *.ts *.webm"], ["All Files", "*.*"]])
    if (isinstance(filename, str)):
        setattr(dataSet, "filename", filename)
        openvideo_btn.text = os.path.basename(filename)
        openvideo_btn.bg = "#cccc00"
        try:
            fExt = os.path.splitext(filename)[1]
            setattr(dataSet, "fileext", fExt)
        except:
            pass

        keyFrameThread = Thread(target = thread_getFrames, args=[filename])
        keyFrameThread.setDaemon(True)
        keyFrameThread.start()
        try:
            csock.sendto(str.encode('{{"event":"filechange","filename":"{}"}}'.format(filename)), ssock_file)
        except:
            print("Cannot send to sdobox.socket")

    else:
    # if (isinstance(filename, (unicode, tuple))):
        openvideo_btn.text = "Open Video"
        openvideo_btn.bg = "#cc33cc"

def playButton():
    if (hasattr(dataSet, "playing") and getattr(dataSet, "playing") == "1"):
        return

    if (hasattr(dataSet, "filename")):
        mpv_player.loop = False
        mpv_player.play(getattr(dataSet, "filename"))
        try:
            csock.sendto(str.encode('{"event":"playing"}'), ssock_file)
        except:
            print("Cannot send playing to sdobox.socket")

class DataSet(object):
    pass

class TeamBtns(object):
    pass

class RoundBtns(object):
    pass

r = RoundBtns()
dataSet = DataSet()
selectedTeam = ""
selectedRound = 0

mpv_player = MPV(start_mpv=False, ipc_socket="/tmp/mpv.socket")
monitorList = get_monitors()
for m in monitorList:
    print(str(m))

appWidth = 720 # monitorList[0].width
appHeight = 480 # monitorList[0].height
appMulti = appWidth / 640
if (appMulti > 1):
    fontSize = int(26 * (appMulti * .85))
    fontSizeMin = int(16 * (appMulti * .85))
else:
    fontSize = 26
    fontSizeMin = 16

app = App(title="Skydive or Bust Dubbing Station", width=appWidth, height=appHeight, layout="grid")

# disc_combo = Combo(app, options=["Select Discipline", "2way Sequential ProAm", "2way Sequential Open", "4way Sequential Open", "4way Rotations"], command=teamChanged, grid=[0,1,8,1], align="top", width="23")
# disc_combo.text_size = fontSize
# disc_combo.bg = "#cc3333"

# team_combo = Combo(app, options=["Select Team", "Team 1", "Team 2", "Team 3", "Team 4"], command=teamChanged, grid=[0,2,6,1], align="top", width="23")
# team_combo.text_size = fontSize
# team_combo.bg = "#33cc33"

openvideo_btn = PushButton(app, command=openButton, text="Open Video", width="17", grid=[0,4,3,1])
openvideo_btn.text_size = fontSizeMin
openvideo_btn.bg = "#cc33cc"

playvideo_btn = PushButton(app, command=playButton, text="Play Video", width="17", grid=[3,4,3,1])
playvideo_btn.text_size = fontSizeMin

# for x in range(0, 6):
#     rX = PushButton(app, command=selectButton, args=[x], text=str(x + 1), grid=[x, 5])
#     rX.text_size = fontSize
#     rX.width = 3
#     rX.bg = "white"
#     setattr(r, "round_{}".format(str(x)), rX)

# for x in range(6, 10):
#     rX = PushButton(app, command=selectButton, args=[x], text=str(x + 1), grid=[x - 6, 6])
#     rX.text_size = fontSize
#     rX.width = 3
#     rX.bg = "white"
#     setattr(r, "round_{}".format(str(x)), rX)

update_btn = PushButton(app, command=submitButton, args=[selectedTeam, selectedRound], text="Upload Video", grid=[0,7,3,1])
update_btn.text_size = fontSize
update_btn.disabled = True

playnew_btn = PushButton(app, command=playnewVideo, text="Play New Video", grid=[4,7,3,1])
playnew_btn.text_size = fontSize
playnew_btn.disabled = True

# move_btn = PushButton(app, command=submitButton, args=[selectedTeam, selectedRound], text="Move Video", grid=[3,7,3,1])
# move_btn.text_size = fontSize
# move_btn.disabled = True

statusText = Text(app, text="", align="left", size=14, grid=[0,10,7,1])

# Bind to key press events with a decorator
@mpv_player.on_event("start-file")
def evStartFile_handler(evData):
    setattr(dataSet, "playing", "1")

@mpv_player.on_event("end-file")
def evStartFile_handler(evData):
    setattr(dataSet, "playing", "0")

ssock_file = '/tmp/sdobox.socket';
csock_file = '/tmp/sdobox.dubbing.socket';

if os.path.exists(csock_file):
    os.remove(csock_file)

csock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
csock.bind(csock_file)
csock.setblocking(0)
csock.sendto(str.encode('{"event":"started"}'), ssock_file)

sdobThread = Thread(target = thread_sdobSocket)
sdobThread.setDaemon(True)
sdobThread.start()

app.when_clicked = globalClicked
app.display()

sdobSocketKiller = 0;

try:
    mpv_player.terminate()
except:
    pass

try:
    csock.sendto(str.encode('{"event":"stopped"}'), ssock_file)
except:
    pass
finally:
    csock.close()

if os.path.exists(csock_file):
    os.remove(csock_file)

sdobThread.join()