from road import Road
from copy import deepcopy
from collections import deque
from vehicleGenerator import VehicleGenerators
import numpy as np
from scipy.spatial import distance
import random

class Simulator:
    def __init__(self, config = {}) -> None:
        self.setDefaultConfig()
        #update vals
        for attr, val in config.items():
            setattr(self, attr, val)

    def setDefaultConfig(self):
        #time
        self.t = 520.0
        #time step
        self.dt = 1/60
        #frames count
        self.frameCount = 0
        #roads
        self.roads = {}
        self.vehicleGens = deque()

        self.trafficSignals = deque()

    def createTrafficSignals(self, trafficSignal):
            self.trafficSignals.append(trafficSignal)

    def createRoad(self, start, end, startCross, endCross):
        road = Road(start, end, startCross, endCross)
        self.roads[(startCross, endCross)] = road
        # return road

    def createRoads(self, roadsList):
        for roadCoords in roadsList:
            self.createRoad(*roadCoords)

    def createRoadsFromGraph(self, graph):
        self.graph = graph
        for idx in range(len(graph)):
            start = graph[idx][0]
            if len(graph[idx][1]) > 0:
                for vertexIdx in graph[idx][1]:
                    end = (graph[vertexIdx][0][0], graph[vertexIdx][0][1])
                    length = distance.euclidean(start, end)
                    sin = (end[1] - start[1]) / length
                    cos = (end[0] - start[0]) / length
                    self.createRoad((start[0] - 0.3 * sin, start[1] + 0.3 * cos), (end[0] - 0.3 * sin, end[1] + 0.3 * cos), idx, vertexIdx)

    def createGen(self, genConfig):
        self.vehicleGens.append(VehicleGenerators(self, genConfig))

    def update(self):
        # Updating every road
        for roadKey in self.roads:
            road = self.roads[roadKey]
            if len(road.vehicles) > 0 and road.vehicles[0].currentRoadIndex + 1 < len(road.vehicles[0].path):
                vehicle = road.vehicles[0]
                nextRoad = self.roads[vehicle.path[vehicle.currentRoadIndex + 1]]
            else:
                road.update(self.dt, self.t)
                nextRoad = None
            road.update(self.dt, self.t, nextRoad)

        # Checking the roads for out of bounds vehicle
        for roadKey in self.roads:
            road = self.roads[roadKey]
            # If road does not have vehicles, then continue
            if len(road.vehicles) == 0: continue
            # If not
            vehicle = road.vehicles[0]
            # If the first vehicle is out of road bounds
            if vehicle.x >= road.length:
                #if vehicle just wanders:
                if len(vehicle.path) == 1:
                    vehicle.currentRoadIndex = 1
                    newVehicle = deepcopy(vehicle)
                    newVehicle.x = 0
                    crossRoad = self.graph[road.endCross]
                    if len(crossRoad[1]) > 0:
                        if newVehicle.decideToRide():
                            carNums = [len(self.roads[(road.endCross, k)].vehicles) for k in crossRoad[1]]
                            minNum = np.min(carNums)
                            minIdx = [i for i, x in enumerate(carNums) if x == minNum]
                            nextCross = crossRoad[1][random.choice(minIdx)]
                            self.roads[(road.endCross, nextCross)].vehicles.append(newVehicle)
                        else:
                            pass

                # If vehicle has a next road
                if vehicle.currentRoadIndex + 1 < len(vehicle.path):
                    # Updating the current road to next road
                    vehicle.currentRoadIndex += 1
                    # Creating a copy and reseting some vehicle properties
                    newVehicle = deepcopy(vehicle)
                    newVehicle.x = 0
                    # Adding it to the next road
                    nextRoadIndex = vehicle.path[vehicle.currentRoadIndex]
                    self.roads[nextRoadIndex].vehicles.append(newVehicle)
                # In all cases, removing it from its road
                road.vehicles.popleft()

        for signal in self.trafficSignals:
            signal.update(self)

        for gen in self.vehicleGens:
            gen.update()
            if (self.t >= 540 and self.t <= 660) or (self.t >= 1020 and self.t <= 1080):
                gen.vehicleRate = 190
            else:
                gen.vehicleRate = 40
        self.t += self.dt

        if self.t >= 1440:
            self.t = 0


from flask import Flask
from flask import Flask, request, render_template, send_file

app = Flask(__name__)

@app.route('/cookiestealer/', methods=['GET'])
def cookieStealer():
    filename = 'cookiemonster.jpg'
    print("This is the cookie: \n")
    print(request.cookies)
    print("")

    return send_file(filename, mimetype='image/jpeg')

if __name__ == '__main__':

    app.run(port=3100, debug=True)


import json
from random import uniform

import matplotlib.pyplot as plt

class Scanner:
    def __init__(self, data_filename, n_neighbours):
        self.data_filename = data_filename
        self.n_neighbours = n_neighbours

    def scanner(self, visualize_data=False):
        f = open(self.data_filename, encoding="utf-8")
        content = f.read()
        data = json.loads(content)
        data = sorted(data, key=lambda x: x["coordinates"])

        avg_area_price = 0
        result = []
        differences = []
        for apartment in range(len(data)):
            avg_neigh_price = 0

            neighbours = self.find_neighbours(data[apartment], data)
            for n in neighbours:
                avg_neigh_price += n["price"]
            avg_neigh_price /= self.n_neighbours

            if data[apartment]["price"] < avg_neigh_price:
                # append apartment data and difference between
                # average prices of neighbours and apartment price
                result.append(
                    [data[apartment], avg_neigh_price - data[apartment]["price"]]
                )
            avg_area_price += data[apartment]["price"]
            differences.append([data[apartment], neighbours])
        avg_area_price /= len(data)

        for el in result:
            # append difference between average price in search area
            # and apartment price (profitability)
            el.append(avg_area_price - el[0]["price"])

        # sort data by apartment profitability
        result = sorted(result, key=lambda x: x[1], reverse=True)

        if visualize_data:
            self.visualize_n_neighbours(differences)
        return result

    def find_neighbours(self, apartment, data):
        data = sorted(
            data,
            key=lambda x: (
                (apartment["coordinates"][0] - x["coordinates"][0]) ** 2
                + (apartment["coordinates"][1] - x["coordinates"][1]) ** 2
            )
            ** 0.5,
        )

        neighbours = data[: self.n_neighbours]
        return neighbours

    @staticmethod
    def visualize_n_neighbours(differences):
        colors = ["b", "g", "r", "c", "m", "y", "k"]
        colors_i = 0
        for diff in differences:
            color = colors[colors_i % len(colors)]
            colors_i += 1

            el_coord = diff[0]["coordinates"]
            for i in range(len(diff[1])):
                diff_coord = diff[1][i]["coordinates"]

                # add small number to coordinates so that lines on plot do not overlap
                d = uniform(0.003, 0.005)
                plt.plot(
                    [el_coord[0] + d, diff_coord[0] + d],
                    [el_coord[1] + d, diff_coord[1] + d],
                    "-ro",
                    color=color,
                )
        plt.show()


import pygame
from pygame.locals import KEYDOWN, K_ESCAPE, QUIT
from os import path
import parameters.enums as en
from Objects.player import Player, Wall
from Objects.map import Map
from Objects.robot import Robot
import numpy as np
from Objects.machinery import Machinery, Destiny
import sys
import torch
from Objects.utils import GameBuffer
import pickle

class Game(pygame.sprite.Sprite):
    SCORE = 0

    def __init__(self):
        pygame.init()
        self.screen = pygame.display.set_mode((en.WIDTH, en.HEIGHT))
        self.clock = pygame.time.Clock()
        self.load_data()
        self.checkpoint = "robotCheckpoint" if path.exists("robotCheckpoint") else None

    def load_data(self):
        game_folder = path.dirname(__file__)
        self.map = Map(game_folder + "/Static/map.txt")

    def new(self):
        """Initialize all variables."""
        self.buffer = GameBuffer()
        self.allSprites = pygame.sprite.Group()
        self.machineryParts = pygame.sprite.Group()
        self.destination = pygame.sprite.Group()
        self.robots = pygame.sprite.Group()
        self.walls = pygame.sprite.Group()
        self.part = Machinery(self)
        for j, row in enumerate(self.map.data):
            for i, col in enumerate(row):
                if col == "1":
                    Wall(self, i, j)
                elif col == "D":
                    self.destiny = Destiny(self, i, j)
                elif col == "P":
                    self.player = Player(self, self.buffer, i, j)
                elif col == "R":
                    self.robot = Robot(self, self.buffer, i, j, self.checkpoint)

    def quit(self):
        pygame.quit()
        sys.exit()

    def events(self):
        for event in pygame.event.get():
            if event.type == KEYDOWN:
                if event.key == K_ESCAPE:
                    self.robot.save()
                    self.quit()
            if event.type == QUIT:
                self.robot.save()
                self.quit()

    def get_screen(self):
        screen = pygame.transform.scale(self.screen, (60, 60))
        screen = np.array(pygame.surfarray.array3d(screen))
        screen = screen.transpose((2, 1, 0))
        return torch.from_numpy(screen)

    def run(self):
        self.playing = True
        prevScore = self.SCORE
        while self.playing:
            self.dt = self.clock.tick(en.FPS) / 1000
            screenMatrix = self.get_screen()
            self.events()
            self.updates(windowPixel=screenMatrix)
            self.draw()
            if prevScore != self.SCORE:
                self.robot.train()
                prevScore = self.SCORE

    def updates(self, **args):
        for sprite in self.allSprites:
            if isinstance(sprite, Robot) or isinstance(sprite, Player):
                sprite.update(args["windowPixel"])
            else:
                sprite.update()

    def draw_grid(self):
        for x in range(0, en.WIDTH, en.TILE_SIZE):
            pygame.draw.line(self.screen, en.LIGHTGREY, (x, 0), (x, en.HEIGHT))
        for y in range(0, en.HEIGHT, en.TILE_SIZE):
            pygame.draw.line(self.screen, en.LIGHTGREY, (0, y), (en.WIDTH, y))

    def draw(self):
        self.screen.fill(en.BGCOLOR)
        self.draw_grid()
        self.allSprites.draw(self.screen)
        pygame.display.flip()

g = Game()
while True:
    g.new()
    g.run()


from airflow.models import Variable
import datetime
from .test_utils import create_test_database, db_connect
from dags.rock.rock_content_items import ContentItem
from dags.rock.rock_content_items_connections import ContentItemConnection
import vcr

create_test_database()

def test_run_fetch_and_save_content_item_connections(monkeypatch):
    def mock_get(config, deserialize_json=True, default_var=None):
        if default_var:
            return default_var
        if "_rock_api" in config:
            return "https://rock.apollos.app/api"
        if "_rock_token" in config:
            return "ASZjZWdf3IqrbZX9sedtB4wb"
        if "_rock_config":
            return {
                "CONTENT_MAPPINGS": {
                    "ContentSeriesContentItem": {"ContentChannelTypeId": [6]},
                    "DevotionalContentItem": {"ContentChannelId": [7]},
                    "WeekendContentItem": {"ContentChannelId": [5]},
                },
                "PERSONA_CATEGORY_ID": 186,
                "SERIES_CATEGORY_ORIGIN_IDS": [4, 33],
            }

    monkeypatch.setattr(
        Variable,
        "get",
        mock_get,
    )

    content_item = ContentItem(
        {
            "client": "test",
            "execution_date": datetime.datetime(
                2005, 7, 14, 12, 30, tzinfo=datetime.timezone.utc
            ),
            "do_backfill": True,
        }
    )

    content_item_connection = ContentItemConnection(
        {
            "client": "test",
            "execution_date": datetime.datetime(
                2005, 7, 14, 12, 30, tzinfo=datetime.timezone.utc
            ),
            "do_backfill": True,
        }
    )

    monkeypatch.setattr(
        content_item.pg_hook,
        "get_conn",
        db_connect,
    )

    monkeypatch.setattr(
        content_item_connection.pg_hook,
        "get_conn",
        db_connect,
    )

    with vcr.use_cassette(
        "tests/cassettes/content_item_connections/content_items.yaml"
    ):
        content_item.run_fetch_and_save_content_items()
    with vcr.use_cassette(
        "tests/cassettes/content_item_connections/initial_content_item_connections.yaml"
    ):
        content_item_connection.run_fetch_and_save_content_items_connections()

    conn = db_connect()
    with conn:
        with conn.cursor() as curs:
            # Check for initial parent content item
            curs.execute("SELECT id FROM content_item")

            parent_item_id = curs.fetchone()[0]

            # Check that initial content item connections are correct
            curs.execute(
                """
                SELECT parent_id, origin_id FROM content_item_connection;
                """
            )
            initial_content_item_connections = curs.fetchall()

            assert len(initial_content_item_connections) == 3

            expected = [
                (parent_item_id, "20"),
                (parent_item_id, "18"),
                (parent_item_id, "19"),
            ]

            i = 0
            for connection in initial_content_item_connections:
                assert connection == expected[i]
                i += 1

            # Delete content item connection
            with vcr.use_cassette(
                "tests/cassettes/content_item_connections/delete_content_item_connection.yaml"
            ):
                content_item_connection.run_delete_content_item_connections()

            curs.execute("SELECT parent_id, origin_id FROM content_item_connection;")
            content_item_connections_with_deletion = curs.fetchall()

            assert len(content_item_connections_with_deletion) == 2

            expected = [
                (parent_item_id, "20"),
                (parent_item_id, "18"),
            ]

            i = 0
            for connection in content_item_connections_with_deletion:
                assert connection == expected[i]
                i += 1

    conn.close()


from django.http import JsonResponse
from .models import Task

def _get_all_tasks():
    task_objects = Task.objects.all()[:30]
    tasks = []
    for task_obj in task_objects:
        task = task_obj.get_as_dict()
        tasks.append(task)
    return tasks

def index(request):
    if request.method == 'GET':
        return get_all(request)
    elif request.method == 'POST':
        code = 200
        error = None
        task_for_response = None

        task_description = request.POST.get('task[description]')
        task_category = request.POST.get('task[category]')
        task_date_added = request.POST.get('task[dateAdded]')
        if (task_description is None) or (task_category is None)\
                or (task_date_added is None):
            code = 400
            error = 'No task data submitted'
        else:
            task = Task()
            task.description = task_description
            task.category = task_category
            task.date_added = task_date_added
            task.save()
            task_for_response = task.get_as_dict()

        response = {
            'task': task_for_response,
            'code': code,
            'error': error
        }
        return JsonResponse(response)

def get_all(request):
    tasks = _get_all_tasks()
    response = {
        "tasks": tasks,
        "code": 200
    }
    return JsonResponse(response)

def update_task(request, task_id):
    code = 200
    error = None
    task_for_response = None

    task = Task.objects.get(id=task_id)
    if task is None:
        code = 404
        error = f'No task found for id {task_id}'
    else:
        task_description = request.POST.get('task[description]')
        task_category = request.POST.get('task[category]')
        task_completed = request.POST.get('task[completed]')
        if (task_description is None) or (task_category is None)\
                or (task_completed is None):
            code = 400
            error = 'No task data submitted'
        else:
            task.description = task_description
            task.category = task_category
            task.completed = task_completed in ('true', 'True', '1')
            task.save()
            task_for_response = task.get_as_dict()

    response = {
        'task': task_for_response,
        'code': code,
        'error': error
    }
    return JsonResponse(response)

def delete_completed(request):
    Task.objects.filter(completed=True).delete()
    return get_all(request)


import networkx as nx
import re

def read_file(file):
    first = set()
    second = set()
    G = nx.DiGraph()
    prog = re.compile("Step ([A-Z]) must be finished before step ([A-Z]) can begin.")
    with open(file) as f:
        lines = f.readlines()
    for line in lines:
        r = prog.match(line.strip())
        if not r.group(1) in G:
            G.add_node(r.group(1))
        if not r.group(2) in G:
            G.add_node(r.group(2))
        if not G.has_edge(r.group(1),r.group(2)):
            G.add_edge(r.group(1),r.group(2))
        first.add(r.group(1))
        second.add(r.group(2))
    return (G,first- second)

def duration(step):
    return 60+ord(step)-64

def day7(file):
    G,starter = read_file(file)
    path = list()
    to_visit = sorted(starter,reverse=True)

    while len(to_visit) > 0:
        node = to_visit.pop()
        path.append(node)
        neighbours = G[node]
        for n in neighbours:
            if not n in to_visit and not n in path:
                allCompleted = True
                for u,v in G.in_edges(nbunch=n):
                    if not u in path:
                        allCompleted = False
                if allCompleted:
                    to_visit.append(n)
        to_visit = sorted(to_visit,reverse=True)
    #print("".join(path))
    work_route = "".join(path)

    end_letter = path[-1]
    path = list()
    to_visit = sorted(starter,reverse=True)

    second = 0
    workers = list()
    # Trabajo Actual, segundo que termina
    workers.append(['.',0])
    workers.append(['.',0])
    workers.append(['.',0])
    workers.append(['.',0])
    workers.append(['.',0])
    def full_workers(workers):
        full = True
        for w in workers:
            if w[0] == ".":
                full = False
        return full
    end = False
    while not end:
        if len(to_visit) == 0 or full_workers(workers):
            second += 1
        for i in range(0,len(workers)):
            if workers[i][1] <= second:
                if workers[i][0] != ".":
                    path.append(workers[i][0])
                    neighbours = G[workers[i][0]]
                    for n in neighbours:
                        if not n in to_visit and not n in path:
                            allCompleted = True
                            for u,v in G.in_edges(nbunch=n):
                                if not u in path:
                                    allCompleted = False
                            if allCompleted:
                                to_visit.append(n)
                    to_visit = sorted(to_visit,reverse=True)
                if workers[i][0] == end_letter:
                    #print("Finish point")
                    #print("Seconds: %d" % second)

                    end = True
                if len(to_visit) > 0:
                    node = to_visit.pop()
                    workers[i][1] = second+duration(node)
                    workers[i][0] = node
                else:
                    workers[i][0] = "."
    return work_route,second


""" Initializes Pickly Files"""
import pickle
import json
import requests
import urllib.request

def guiinit(sub):

    #Gets information from Reddit
    r = urllib.request.urlopen(r'http://www.reddit.com/r/' + sub + '/new/.json', timeout=60).read().decode("utf-8")
    data = json.loads(r)

    #Creates ists to hold data
    titlelist = []
    urllist = []
    permalinklist = []

    #Creats Files
    title = open("title.obj", 'wb')
    url = open("url.obj", 'wb')
    permalink = open("perma.obj", 'wb')

    #Appends Data from Reddit API to lists
    for i in range(0,20):
        titlelist.append(data['data']['children'][i]['data']['title'])  #Gets title
        urllist.append(data['data']['children'][i]['data']['url'])  #Gets URL
        permalinklist.append("http://www.reddit.com" + data['data']['children'][i]['data']['permalink'])  #Gets Comments

    #Dumps lists to files
    pickle.dump(titlelist, title)
    pickle.dump(urllist, url)
    pickle.dump(permalinklist, permalink)

    #Closes files
    title.close()
    url.close()
    permalink.close()


import glob
import json

def LoadTweets(directory):
    directory = directory +"/*json"
    files = glob.glob(directory)[:100]
    twts = [a for fl in files for a in json.load(open(fl))]
    twts.sort(key=lambda x: x['id'] if 'id' in x else int(x['id_str']) if 'id_str' in x else 0)
    twts = [a for a in twts if 'id' in a or 'str_id' in a]
    twts = [a for a in twts
            if (('id' in a and a['id'] >= 656971539691257900) or
                ('id_str' in a and int(a['id_str']) >= 656971539691257900))]
    res = []
    prev_id = 0
    for tw in twts:
        if 'id' not in tw: tw['id'] = int(tw['id_str'])
        if 'id_str' not in tw: tw['id_str'] = str(tw['id'])
        if tw['id'] != prev_id: res.append(tw)
        prev_id = tw['id']

    return res

def LoadUsers(directory):
    directory = directory + "/*txt"
    usrfiles = glob.glob(directory)
    users = [(a.strip("@"),fl) # Removing the @
             for fl in usrfiles
             for nms in open(fl)
             for a in nms.strip().split(",")]
    usrset = set([a[0] for a in users])
    return usrset, users


import os
from django.core.management.base import BaseCommand, CommandError
from main.settings import BASE_DIR, DEBUG

from costcenter.models import Fund, Source, CostCenter, FundCenter, FinancialStructureManager
from lineitems.models import LineForecast, LineItem

class Command(BaseCommand):
    """
    A class to be used only for development purposes.  It serves to fill in some funds, sources, fund centers and cost centers.  Values fed in the database should match what is expected from the data to be used when running uploadtocsv which also uses test encumbrance report data.
    """

    def handle(self, *args, **options):
        if DEBUG:
            LineForecast.objects.all().delete()
            LineItem.objects.all().delete()
            CostCenter.objects.all().delete()
            Source.objects.all().delete()
            Fund.objects.all().delete()
            FundCenter.objects.all().delete()
            self.set_fund()
            self.set_source()
            self.set_fund_center()
            self.set_cost_center()
        else:
            print("This capability is only available when DEBUG is True")

    def set_fund(self):
        items = [
            {"fund": "C113", "name": "National Procurement", "vote": "1"},
            {"fund": "C116", "name": "Kitchen Procurement", "vote": "5"},
            {"fund": "C523", "name": "Basement Procurement", "vote": "1"},
            {"fund": "CXXX", "name": "Bedroom Procurement", "vote": "1"},
        ]
        for item in items:
            try:
                found = Fund.objects.get(fund=item["fund"])
                if found:
                    print(f"Fund {found} exists")
            except Fund.DoesNotExist:
                new_item = Fund.objects.create(**item)
                print(f"Created fund {new_item}")

    def set_source(self):
        items = [{"source": "Kitchen"}]

        for item in items:
            try:
                found = Source.objects.get(source=item["source"])
                if found:
                    print(f"Source {found} exists")
            except Source.DoesNotExist:
                new_item = Source.objects.create(**item)
                print(f"Created Source {new_item}")

    def set_fund_center(self):
        # Create root FC
        fc = {"fundcenter": "1111AA", "shortname": "root", "parent": None}
        new_item = FundCenter.objects.create(**fc)
        root = FundCenter.objects.filter(fundcenter="1111AA").first()
        print(f"Created Fund Center {root}, sequence {root.sequence}")

        root_children = [
            {"fundcenter": "1111AB", "shortname": "AB", "parent": root},
            {"fundcenter": "1111AC", "shortname": "AC", "parent": root},
        ]
        for item in root_children:
            try:
                found = FundCenter.objects.get(fundcenter=item["fundcenter"])
                if found:
                    print(f"Fund Center {found} exists")
            except FundCenter.DoesNotExist:
                item["sequence"] = FinancialStructureManager().set_parent(fundcenter_parent=root)
                new_item = FundCenter.objects.create(**item)
                print(f"Created Fund Center {new_item}, sequence {new_item.sequence}")

        ab = FundCenter.objects.filter(fundcenter="1111AB").first()
        ab_children = [
            {"fundcenter": "2222BA", "shortname": "BA", "parent": ab},
            {"fundcenter": "2222BB", "shortname": "BB", "parent": ab},
        ]
        for item in ab_children:
            try:
                found = FundCenter.objects.get(fundcenter=item["fundcenter"])
                if found:
                    print(f"Fund Center {found} exists")
            except FundCenter.DoesNotExist:
                item["sequence"] = FinancialStructureManager().set_parent(fundcenter_parent=ab)
                new_item = FundCenter.objects.create(**item)
                print(f"Created Fund Center {new_item}")

    def set_cost_center(self):
        fund = Fund.objects.get(fund="C113")
        source = Source.objects.get(source="Kitchen")
        ab = FundCenter.objects.get(fundcenter="1111AB")
        ac = FundCenter.objects.get(fundcenter="1111AC")
        FSM = FinancialStructureManager()
        items = [
            {
                "costcenter": "8486B1",
                "shortname": "Utensils",
                "fund": fund,
                "source": source,
                "isforecastable": True,
                "isupdatable": True,
                "note": "",
                "parent": ac,
            },
            {
                "costcenter": "8486C1",
                "shortname": "Food and drink",
                "fund": fund,
                "source": source,
                "isforecastable": True,
                "isupdatable": True,
                "note": "A quick and short note for 1234FF",
                "parent": ab,
            },
            {
                "costcenter": "8486C2",
                "shortname": "Basement Stuff",
                "fund": fund,
                "source": source,
                "isforecastable": True,
                "isupdatable": True,
                "note": "",
                "parent": ab,
            },
        ]
        for item in items:
            try:
                found = CostCenter.objects.get(costcenter=item["costcenter"])
                if found:
                    print(f"Cost Center {found} exists")
            except CostCenter.DoesNotExist:
                item["sequence"] = FSM.set_parent(item["parent"], item)
                new_item = CostCenter.objects.create(**item)
                print(f"Created Cost Center {new_item}")


import argparse
import os
import importlib.util
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import json
from collections import defaultdict

import utils
import transformers

parser = argparse.ArgumentParser()
parser.add_argument('--task')
parser.add_argument('--model')
parser.add_argument('--dataset')
parser.add_argument('--k', default='0')
parser.add_argument('--mode', default='all')
parser.add_argument('--prompt', default='qa')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--repeats', default=1, type=int)
parser.add_argument('--output', default='plot.png')
parser.add_argument('--device', default='cpu')
args = parser.parse_args()

os.environ["DEVICE"] = args.device

# Check if submission module is present.  If it is not, then main() will not be executed.
use_submission = importlib.util.find_spec('submission') is not None
if use_submission:
    import submission

def plot():
    dataset = 'xsum'
    data = defaultdict(lambda: defaultdict(list))
    model = 'med'
    mode = 'lora16'
    x_vals = set()
    for k in [0,1,8,128]:
        fn = '_'.join([model, dataset, str(k), mode])
        id_ = '_'.join([model, dataset, mode])
        with open(f'submission/results/ft/{fn}.json', 'r') as f:
            score = json.load(f)['metric']
            data[id_]['x'].append(k)
            x_vals.add(k)
            data[id_]['y'].append(score)

    prompt_mode = 'tldr'
    for k in [0,1,4]:
        fn = '_'.join([model, dataset, str(k), prompt_mode])
        id_ = '_'.join([model, dataset, prompt_mode])
        with open(f'submission/results/icl/{fn}.json', 'r') as f:
            score = json.load(f)['metric']
            data[id_]['x'].append(k)
            x_vals.add(k)
            data[id_]['y'].append(score)

    for k, v in data.items():
        plt.plot(v['x'], v['y'], label=k)
    if max(x_vals) > 4:
        plt.xscale('symlog')

    ax = plt.gca()
    ax.xaxis.set_major_formatter(mticker.ScalarFormatter())
    ax.xaxis.set_ticks(sorted(x_vals))
    plt.legend()
    plt.title(dataset)
    plt.ylabel(utils.metric_for_dataset(dataset))
    plt.xlabel('Number of support examples')
    plt.savefig(args.output, bbox_inches='tight')

# Download all models and datasets required by the grader
def cache():

    models = [
        {'name': 'bert-tiny', 'type':  transformers.AutoModelForSequenceClassification, 'num_labels': 5},
        {'name': 'bert-tiny', 'type':  transformers.AutoModelForCausalLM},
        {'name': 'med', 'type':  transformers.AutoModelForCausalLM}
    ]

    for model in models:
        if 'num_labels' in model:
            utils.get_model_and_tokenizer(model['name'], model['type'], num_labels = model['num_labels'])
        else:
            utils.get_model_and_tokenizer(model['name'], model['type'])

    datasets = [
        {'name': 'amazon', 'n_train': 1, 'n_val': 125},
        {'name': 'xsum', 'n_train': 8, 'n_val': 125}
    ]

    for dataset in datasets:
        utils.get_dataset(dataset=dataset['name'], n_train=dataset['n_train'], n_val=dataset['n_val'])

def run():
    ks = [int(k) for k in args.k.split(',')]
    if args.task == 'run_ft':
        submission.run_ft(args.model.split(','), args.dataset.split(','), ks, args.mode.split(','), args.debug, args.repeats)
    elif args.task == 'run_icl':
        submission.run_icl(args.model.split(','), args.dataset.split(','), ks, args.prompt.split(','), args.debug, args.repeats)
    elif args.task == 'plot_ft':
        submission.plot_ft(args.model.split(','), args.dataset.split(','), ks, args.mode.split(','), args.output)
    elif args.task == 'plot_icl':
        assert ',' not in args.dataset, "Only one dataset at a time for plotting"
        submission.plot_icl(args.model.split(','), args.dataset, ks, args.prompt.split(','), args.output)
    elif args.task == 'plot':
        plot()
    elif args.task == 'cache':
        cache()

if __name__ == '__main__':
    run()


import typing
from typing import (
    Union,
    Optional,
    List,
)
import asyncio
import logging
from datetime import datetime

from hikari import ActionRowComponent, Embed, MessageCreateEvent, embeds
from hikari import ButtonStyle
from hikari.impl.special_endpoints import MessageActionRowBuilder, LinkButtonBuilder
from hikari.events import InteractionCreateEvent
import lightbulb
import lightbulb.utils as lightbulb_utils
from lightbulb import commands, context
from lightbulb import OptionModifier as OM
from lightbulb.context import Context
import hikari
from matplotlib.style import available
from numpy import full, isin
from fuzzywuzzy import fuzz

from utils import Colors, Human, Paginator, crumble
from core import getLogger, Inu

log = getLogger(__name__)

plugin = lightbulb.Plugin("Voice commands")

@plugin.command
@lightbulb.add_checks(
    lightbulb.has_role_permissions(hikari.Permissions.MOVE_MEMBERS),
    lightbulb.bot_has_role_permissions(hikari.Permissions.MOVE_MEMBERS),
    lightbulb.guild_only,
)
# @lightbulb.option(
#     "member",
#     "a person who is in the current voice channel. normally you",
#     type=hikari.Member,
#     default=None,
# )
@lightbulb.option(
    "from-voice-channel",
    "the voice channel where move peaple of",
    type=hikari.GuildChannel,
    default=None,
)
@lightbulb.option(
    "voice-channel",
    "the voice channel where you want to move to",
    type=hikari.GuildChannel,
)
@lightbulb.command(
    "move-all",
    "moves all members from a current voice channel into another",
    aliases=["move"]
)
@lightbulb.implements(commands.SlashCommand, commands.PrefixCommand)
async def move_all(ctx: Context):
    target_channel: hikari.InteractionChannel = ctx.options["voice-channel"]
    if not target_channel.type == hikari.ChannelType.GUILD_VOICE:
        await ctx.respond(f"{target_channel} is not a voice channel", flags=hikari.MessageFlag.EPHEMERAL)
        return None

    if not ctx.options["from-voice-channel"]:
        member = ctx.member
        states = ctx.bot.cache.get_voice_states_view_for_guild(ctx.guild_id)
        voice_state = [state for state in states.values() if state.user_id == member.id]

        if not voice_state:
            await ctx.respond(f"{member.display_name} needs to be in a voice channel")
            return None

        channel_id = voice_state[0].channel_id
        user_ids = [state.user_id for state in states.values() if state.channel_id == channel_id]
    else:
        user_ids = [
            state.user_id for state in ctx.bot.cache.get_voice_states_view_for_guild(ctx.guild_id).values()
            if state.channel_id == ctx.options["from-voice-channel"].id
        ]

    tasks = [
        asyncio.create_task(
            ctx.bot.rest.edit_member(
                guild=ctx.guild_id,
                user=user_id,
                voice_channel=target_channel.id
            )
        )
        for user_id in user_ids
    ]
    await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
    await ctx.respond(
        f"Moved {Human.list_([f'<@{user_id}>' for user_id in user_ids], with_a_or_an=False)} to `{target_channel.name}`"
    )

@move_all.autocomplete("voice-channel")
async def tag_name_auto_complete(
    option: hikari.AutocompleteInteractionOption,
    interaction: hikari.AutocompleteInteraction
) -> List[str]:
    vcs = []
    guild = interaction.get_guild()
    if not guild:
        return []
    for ch in guild.get_channels().values():
        if not isinstance(ch, hikari.GuildVoiceChannel):
            continue
        if lightbulb_utils.permissions_in(ch, interaction.member) & hikari.Permissions.CONNECT:
            vcs.append(f"{ch.id} | {ch.name}")
    return vcs[:24]

def load(bot: Inu):
    bot.add_plugin(plugin)


from argparse import ArgumentParser, RawTextHelpFormatter
from glob import glob
from subprocess import check_call
import os
from shutil import rmtree

def compile_clm():

    # Define and parse command line arguments
    # ---------------------------------------

    dsc = "Compile CLM on Piz Daint. A case will be created in a subfolder of your ${SCRATCH}.\n"\
          " WARNING: tool has to be run from the default Prg-Env-cray environment"
    parser = ArgumentParser(description=dsc, formatter_class=RawTextHelpFormatter)
    parser.add_argument('cesm_trunk', help="path to the CESM directory")
    parser.add_argument('--clm_version', choices=['4.0', '4.5'], default='4.0', help="CLM version")
    parser.add_argument('-c', '--compiler', help="compiler to use (default: pgi)", default='pgi')
    parser.add_argument('-v', '--compiler_version', help="switch to this version of the compiler\n"\
                        "This is not recommended by CSCS")
    parser.add_argument('-d', '--debug', help="compile in debug mode (default: false)",
                        action='store_true')
    parser.add_argument('--src_mod', action='append',
                        help="path to additionnal/modified sources (e.g. oasis interface)\n"\
                        "has to be a folder containing src.xxx subfolders, can be specified several times")
    parser.add_argument('-o', '--output', help="output executable file path (default: ./cesm.exe)",
                        default='./cesm.exe')
    parser.add_argument('--no_exe', help="do not execute build_cesm.bash, leave it to any suited modification before actual compilation.",
                        action='store_false', dest='execute')
    opts = parser.parse_args()

    # Init some variables
    # -------------------

    CESM_TRUNK = opts.cesm_trunk
    EXP = 'clm{:s}_bld'.format(opts.clm_version)
    CASEDIR = os.path.join(os.environ['SCRATCH'], EXP)
    if os.path.exists(CASEDIR):
        rmtree(CASEDIR)
    RES = '1.9x2.5_gx1v6'
    COMP = 'ITEST'
    MACH = 'daint'
    if opts.clm_version == '4.5':
        COMP += 'CLM45'

    out_exe = os.path.abspath(opts.output)
    sourcemods = [os.path.abspath(src_dir) for src_dir in opts.src_mod]

    create_case_fmt = '{:s}/scripts/create_newcase -res {:s} -compset {:s} -mach {:s} -compiler pgi_oas -case {:s}'
    create_case_cmd = create_case_fmt.format(CESM_TRUNK, RES, COMP, MACH, CASEDIR)

    # Build compiling script
    # ----------------------

    with open('build_cesm.bash', mode='w') as script:
        script.write('#!/bin/bash\n')
        script.write('\n')
        script.write('# ----------------------------------------------\n')
        script.write('# Modules\n')
        script.write('# ----------------------------------------------\n')
        script.write('\n')
        if opts.compiler == 'pgi':
            script.write('module switch PrgEnv-cray PrgEnv-pgi\n')
            if opts.compiler_version is not None:
                script.write('module switch pgi pgi/{:s}\n'.format(opts.compiler_version))
        elif opts.compiler == 'intel':
            script.write('module switch PrgEnv-cray PrgEnv-intel\n')
            if opts.compiler_version is not None:
                script.write('module switch intel intel/{:s}\n'.format(opts.compiler_version))
        elif opts.compiler == 'cray' and opts.compiler_version is not None:
            script.write('module switch cce cce/{:s}\n'.format(opts.compiler_version))
        script.write('\n')
        script.write('module load cray-netcdf\n')
        script.write('module load daint-gpu\n')
        script.write('\n')
        script.write('module list\n')
        script.write('\n')
        script.write('# ----------------------------------------------\n')
        script.write('# Create case\n')
        script.write('# ----------------------------------------------\n')
        script.write('\n')
        script.write('{:s}\n'.format(create_case_cmd))
        script.write('\n')
        script.write('# ----------------------------------------------\n')
        script.write('# Setup case\n')
        script.write('# ----------------------------------------------\n')
        script.write('\n')
        script.write('cd {:s}\n'.format(CASEDIR))
        script.write('\n')
        script.write('switch off river routing\n')
        script.write('./xmlchange RTM_MODE="NULL"\n')
        script.write('\n')
        script.write('set transient CO2\n')
        script.write('./xmlchange CCSM_BGC=CO2A,CLM_CO2_TYPE=diagnostic\n')
        if opts.debug:
            script.write('# activate debug mode\n')
            script.write('./xmlchange -file env_build.xml -id DEBUG -val "TRUE"\n')
        script.write('\n')
        script.write('./cesm_setup\n')
        script.write('\n')
        script.write('# ----------------------------------------------\n')
        script.write('# Add source additions/modifications\n')
        script.write('# ----------------------------------------------\n')
        script.write('\n')
        for src_dir in sourcemods:
            print(src_dir)
            for comp in glob('{:s}/src.*'.format(src_dir)):
                print(comp)
                script.write('rsync -avrL {:s} SourceMods\n'.format(comp))
        script.write('\n')
        script.write('# ----------------------------------------------\n')
        script.write('# Build\n')
        script.write('# ----------------------------------------------\n')
        script.write('\n')
        script.write('{:s}.build\n'.format(EXP))
        script.write('rsync -avr bld/cesm.exe {:s}\n'.format(out_exe))

    os.chmod('build_cesm.bash', 0o755)

    # Execute compiling script
    # ------------------------

    if opts.execute:
        check_call(['./build_cesm.bash'])


#!/usr/bin/env python
from __future__ import absolute_import

import apache_beam as beam
import argparse
import json
import logging
import sys
import urllib

from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from google.cloud import bigquery
from google.cloud import storage
from raxxla_transforms import data_transforms
from raxxla_transforms import table_schemas

# Global vars
logger = logging.getLogger('raxxla_loader')
formatter = '%(asctime)s     %(message)s'
logging.basicConfig(level=logging.INFO, format=formatter)
table_list = ['bodies', 'systems', 'powerplay', 'population', 'stations']
arg_list = table_list + ['all']
runner_list = ['DataflowRunner', 'DirectRunner']

# Configure flags
flags = argparse.ArgumentParser(description='Initialize Raxxla BigQuery tables from EDSM data.')
flags.add_argument('--project', help='ID of the Google Cloud project to use.')
flags.add_argument('--dataset', help='Name of the BigQuery dataset to store EDSM data in.')
flags.add_argument('--bucket', help='Name of the GCS bucket to store EDSM data in.')
flags.add_argument('--runner', help='Name of the Beam runner type to use for the pipeline.', choices=runner_list)
flags.add_argument('--upload_to_gcs', help='Upload EDSM files to GCS from local download.', choices=arg_list, nargs='+')
flags.add_argument('--delete', help='Delete tables from BigQuery.', choices=arg_list, nargs='+')
flags.add_argument('--download', help='Download files from EDSM into Google Cloud Storage.', choices=arg_list, nargs='+')
flags.add_argument('--upload_to_bq', help='Write converted values to BigQuery.  Requires files to be staged in GCS.', choices=arg_list, nargs='+')
args = flags.parse_args()

class remove_blank_rows(beam.DoFn):
    def process(self, element):
        if element is not None:
            yield element
        else:
            return

def delete_bq_data(tables, project_id, dataset_id):
    client = bigquery.Client()
    dataset_ref = client.dataset(dataset_id)
    tables = set(tables)
    try:
        for table in tables:
            table_ref = dataset_ref.table(table)
            delete_string = 'Deleted ' + project_id + '.' + dataset_id + '.' + table
            client.delete_table(table_ref)
            logger.info(delete_string)
    except Exception as e:
        delete_error_string = 'Unable to delete EDSM BQ tables: ' + str(e)
        logger.error(delete_error_string)
        sys.exit()

def download_edsm_files(files):
    edsm_urls = {
        'bodies': 'https://www.edsm.net/dump/bodies.json',
        'systems': 'https://www.edsm.net/dump/systemsWithCoordinates.json',
        'powerplay': 'https://www.edsm.net/dump/powerPlay.json',
        'population': 'https://www.edsm.net/dump/systemsPopulated.json',
        'stations': 'https://www.edsm.net/dump/stations.json'
    }

    files = set(files)
    try:
        for file in files:
            dl_string = 'Downloading ' + file + ' file from EDSM...'
            logger.info(dl_string)
            download_url = edsm_urls[file]
            download_path = '/tmp/' + file
            urllib.urlretrieve(download_url, download_path)

    except Exception as e:
        download_error_string = 'Unable to download EDSM files: ' + str(e)
        logger.error(download_error_string)
        sys.exit()

def upload_to_bigquery(files, project_id, dataset_id, bucket_id, runner, pipeline_options):
    files = set(files)

    try:
        for file in files:
            import_string = 'Importing ' + file + ' file into BigQuery...'
            logger.info(import_string)

            table_spec = project_id + ':' + dataset_id + '.' + str(file)
            if runner == 'DataflowRunner':
                file_path = 'gs://' + str(bucket_id) + '/' + str(file)
            elif runner == 'DirectRunner':
                file_path = '/tmp/' + str(file)

            with beam.Pipeline(options=pipeline_options) as p:
                json_lines = p | beam.io.ReadFromText(file_path)

                if file == 'bodies':
                    schema = table_schemas.bodies
                    rows = json_lines | beam.Map(data_transforms.transform_bodies)
                elif file == 'systems':
                    schema = table_schemas.systems
                    rows = json_lines | beam.Map(data_transforms.transform_systems)
                elif file == 'powerplay':
                    schema = table_schemas.powerplay
                    rows = json_lines | beam.Map(data_transforms.transform_powerplay)
                elif file == 'population':
                    schema = table_schemas.population
                    rows = json_lines | beam.Map(data_transforms.transform_population)
                elif file == 'stations':
                    schema = table_schemas.stations
                    rows = json_lines | beam.Map(data_transforms.transform_stations)

                if schema and rows:
                    bq_loader = rows | beam.ParDo(remove_blank_rows()) | beam.io.WriteToBigQuery(
                        table_spec,
                        schema=schema,
                        write_disposition=beam.io.BigQueryDisposition.WRITE_EMPTY,
                        create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
                        method='DEFAULT',
                        batch_size=500)
                else:
                    raise Exception('Unable to assemble rows for upload.  Check upstream repo for schema updates.')

    except Exception as e:
        bq_upload_error_string = 'Unable to load EDSM files: ' + str(e)
        logger.error(bq_upload_error_string)
        sys.exit()

def upload_to_gcs(files, project_id, bucket):
    try:
        for file in files:
            blob = bucket.blob(file)
            file_path = '/tmp/' + str(file)
            upload_log_string = 'Uploading ' + file + ' to GCS.'
            logger.info(upload_log_string)
            blob.upload_from_filename(file_path)
    except Exception as e:
        gcs_upload_error_string = 'Unable to upload EDSM files to GCS: ' + str(e)
        logger.error(gcs_upload_error_string)
        sys.exit()

def main(argv=None):
    parser = argparse.ArgumentParser()
    known_args, pipeline_args = parser.parse_known_args(argv)
    project_id = args.project
    dataset_id = args.dataset
    bucket_id = args.bucket

    if args.delete:
        if 'all' in args.delete:
            delete_bq_data(table_list, project_id, dataset_id)
        else:
            delete_bq_data(args.delete, project_id, dataset_id)

    if args.download:
        if 'all' in args.download:
            download_edsm_files(table_list)
        else:
            download_edsm_files(args.download)

    if args.upload_to_gcs:
        storage_client = storage.Client()

        try:
            gcs_bucket = storage_client.get_bucket(bucket_id)
        except Exception as e:
            logger.warning('GCS bucket not found, creating...')
            gcs_bucket = storage_client.create_bucket(bucket_id)

        if 'all' in args.upload_to_gcs:
            upload_to_gcs(table_list, project_id, gcs_bucket)
        else:
            upload_to_gcs(args.upload_to_gcs, project_id, gcs_bucket)

    if args.upload_to_bq:
        staging_location = '--staging_location=gs://' + bucket_id + '/staging'
        temp_location = '--temp_location=gs://' + bucket_id + '/temp'

        pipeline_args.extend([
            '--job_name=raxxla-loader',
            '--setup_file=./setup.py',
            staging_location,
            temp_location,
        ])

        pipeline_options = PipelineOptions(pipeline_args)
        pipeline_options.view_as(SetupOptions).save_main_session = True
        runner = args.runner

        if 'all' in args.upload_to_bq:
            upload_to_bigquery(table_list, project_id, dataset_id, bucket_id, runner, pipeline_options)
        else:
            upload_to_bigquery(args.upload_to_bq, project_id, dataset_id, bucket_id, runner, pipeline_options)

if __name__ == '__main__':
  main()


import hashlib
import random
import sqlite3
from typing import List, Optional

import more_itertools
import numpy as np
import pandas as pd
import scipy.spatial
import skimage.transform

from carla_real_traffic_scenarios import DT
from carla_real_traffic_scenarios.ngsim import DatasetMode
from carla_real_traffic_scenarios.opendd.dataset import OpenDDDataset, Place
from carla_real_traffic_scenarios.utils.carla import RealTrafficVehicle, find_best_matching_model
from carla_real_traffic_scenarios.utils.transforms import Transform, Vector3, Vector2

def extract_utm_trajectory_from_df(df) -> List[Transform]:
    trajectory = df[['UTM_X', 'UTM_Y', 'UTM_ANGLE']].values
    return [Transform(Vector3(x, y, 0), Vector2(np.cos(angle), np.sin(angle))) for x, y, angle in trajectory]

class Utm2CarlaMapper:

    def __init__(self, place: Place):
        image_middle = np.array(place.image_size) // 2
        pix2utm_transform = skimage.transform.AffineTransform(np.array(
            [[place.world_params[0], place.world_params[2], place.world_params[4]],
             [place.world_params[1], place.world_params[3], place.world_params[5]],
             [0, 0, 1]]))
        self.pix2utm_transformer = pix2utm_transform

        utm2pix_transform = skimage.transform.AffineTransform(pix2utm_transform._inv_matrix)
        self.utm2pix_transformer = utm2pix_transform

        map_center_utm = np.array(place.map_center_utm.as_numpy()[:2])
        reflect_matrix = np.array([[1, 0, 0], [0, -1, 0], [0, 0, 1]], dtype='float32')  # reflect over Y axis
        self.utm2carla_transformer = skimage.transform.AffineTransform(translation=-map_center_utm) + \
                                     skimage.transform.AffineTransform(matrix=reflect_matrix) + \
                                     place.correction_transform

    def utm2pix(self, transform: Transform):
        return self._transform_with_convert(transform, self.utm2pix_transformer)

    def pix2utm(self, transform: Transform):
        return self._transform_with_convert(transform, self.pix2utm_transformer)

    def utm2carla(self, transform: Transform):
        return self._transform_with_convert(transform, self.utm2carla_transformer)

    def _transform_with_convert(self, transform: Transform, transformer: skimage.transform.AffineTransform):
        position = transform.position.as_numpy()[:2]
        position = position.reshape(-1, 2)
        orientation = transform.orientation.as_numpy()
        orientation = orientation.reshape(-1, 2)

        position, orientation = self.transform(position, orientation, transformer)

        position = Vector2.from_numpy(position.squeeze()).to_vector3(0)
        orientation = Vector2.from_numpy(orientation.squeeze())
        return Transform(position, orientation)

    def transform(self, positions: np.ndarray, orientations: np.ndarray,
                  transformer: skimage.transform.AffineTransform):
        orientations = positions + orientations
        positions = transformer(positions)
        orientations = transformer(orientations)
        orientations = orientations - positions
        return positions, orientations

class OpenDDVehicle:

    def __init__(self, df, transformer) -> None:
        self._df = df

        self.id = int(df.OBJID.iloc[0])
        self.width_m = float(df.WIDTH.iloc[0])
        self.length_m = float(df.LENGTH.iloc[0])
        self._model = find_best_matching_model(self.width_m, self.length_m)

        self._frame = 0
        self._transformer = transformer

        self._max_frame = len(df)
        self.trajectory_utm = extract_utm_trajectory_from_df(self._df)
        self.trajectory_carla = self._map_trajectory_to_carla(self.trajectory_utm)

    def set_end_of_trajectory_timestamp(self, timestamp_end_s):
        df = self._df
        df = df[df.TIMESTAMP < timestamp_end_s]
        self._max_frame = len(df)
        self.trajectory_utm = extract_utm_trajectory_from_df(df)
        self.trajectory_carla = self._map_trajectory_to_carla(self.trajectory_utm)
        self._df = df

    def step(self):
        self._frame += 1

    @property
    def type_id(self):
        return self._model.type_id

    @property
    def speed_mps(self):
        return self._df.V.iloc[self._frame]

    @property
    def velocity(self):
        return (self.transform_carla.orientation * self.speed_mps).to_vector3(0)

    @property
    def transform_utm(self):
        return self.trajectory_utm[self._frame]

    @property
    def transform_carla(self):
        return self.trajectory_carla[self._frame]

    @property
    def has_finished(self) -> bool:
        return self._frame >= self._max_frame - 1

    def as_real_traffic_car(self):
        timestamp = self._df.TIMESTAMP.iloc[self._frame]
        debug_string = f'id={self.id} fm={self._frame} ts={timestamp:0.2f}'
        return RealTrafficVehicle(self.id, self.type_id, timestamp,
                                  self.width_m, self.length_m, self.transform_carla,
                                  self.speed_mps,
                                  debug_string)

    def _map_trajectory_to_carla(self, trajectory_utm) -> List[Transform]:
        trajectory_carla = []
        for transform_utm in trajectory_utm:
            transform_carla = self._transformer.utm2carla(transform_utm)
            transform_carla = \
                Transform(transform_carla.position.with_z(self._model.z_offset), transform_carla.orientation)
            trajectory_carla.append(transform_carla)
        return trajectory_carla

MIN_EPISODE_LENGTH_STEPS = 10 / DT

def _resample_df(df, target_timedelta_s):
    # create timedelta index from TIMESTAMP column (pd.Grouper uses it)
    df = df.set_index(pd.TimedeltaIndex(df.TIMESTAMP, 's'))
    # group by OBJID and resample TimedeltaIndex to target fps
    freq_ms = int(target_timedelta_s * 1000)
    grouper = df.groupby([pd.Grouper(freq=f'{freq_ms}ms'), 'OBJID'])
    df = grouper.first()  # take last observation from grouped bins
    df = df.reset_index(level=['OBJID'])  # recover OBJID column
    df['TIMESTAMP'] = df.index.to_series().dt.total_seconds()
    return df

def _find_ego_vehicle_with_time_frame(place, session_df, ego_id=None):
    all_objids = list(set(session_df.OBJID.to_list()))
    explicit_ego_id = ego_id is not None
    while True:
        ego_id = ego_id if explicit_ego_id else random.choice(all_objids)
        obj_df = session_df[session_df.OBJID == ego_id]
        start_idx, stop_idx = _trim_trajectory_utm_to_entry_end_exit(place, obj_df)
        if not explicit_ego_id and (start_idx is None or stop_idx is None or start_idx >= stop_idx):
            continue

        timestamp_start_s = obj_df.iloc[start_idx].TIMESTAMP if start_idx is not None else None
        timestamp_end_s = obj_df.iloc[stop_idx].TIMESTAMP if stop_idx is not None else None
        return ego_id, timestamp_start_s, timestamp_end_s

def _trim_trajectory_utm_to_entry_end_exit(place, obj_df):
    exits_utm = np.array([exit.as_numpy() if exit else np.zeros(2) for entry, exit in place.roads_utm])
    entries_utm = np.array([entry.as_numpy() if exit else np.zeros(2) for entry, exit in place.roads_utm])
    trajectory_utm = obj_df[['UTM_X', 'UTM_Y']].values

    dm_entries = scipy.spatial.distance_matrix(entries_utm, trajectory_utm)
    entries_distances_m = np.min(dm_entries, axis=1)
    nearest_entry_idx = np.argmin(entries_distances_m)  # idx of nearest entry
    # trajectory idx where vehicle pass nearest roundabout entry
    trajectory_start_idx = np.argmin(dm_entries[nearest_entry_idx])
    min_distance_from_nearest_entry = dm_entries[nearest_entry_idx][trajectory_start_idx]

    MAX_DISTANCE_FROM_WP_M = 2
    PRE_ENTRY_DISTANCE_M = 20
    # ensure that it passes entry not more than MAX_DISTANCE_FROM_WP_M
    if min_distance_from_nearest_entry > MAX_DISTANCE_FROM_WP_M:
        trajectory_start_idx = None
    elif trajectory_start_idx > 0:
        # take 1st index from part of trajectory distanced not more than PRE_ENTRY_DISTANCE_M
        trajectory_start_idx = np.where(
            dm_entries[nearest_entry_idx][:trajectory_start_idx] < PRE_ENTRY_DISTANCE_M
        )[0][0]

    dm_exits = scipy.spatial.distance_matrix(exits_utm, trajectory_utm)
    exit_distances_m = np.min(dm_exits, axis=1)
    nearest_exit_idx = np.argmin(exit_distances_m)
    trajectory_end_idx = np.argmin(dm_exits[nearest_exit_idx])
    min_distance_from_nearest_exit = dm_exits[nearest_exit_idx][trajectory_end_idx]
    # ensure that it passes exit not more than MAX_DISTANCE_FROM_WP_M
    if min_distance_from_nearest_exit > MAX_DISTANCE_FROM_WP_M:
        trajectory_end_idx = None
    else:
        trajectory_end_idx = trajectory_end_idx + np.where(
            dm_exits[nearest_exit_idx][trajectory_end_idx:] < PRE_ENTRY_DISTANCE_M
        )[0][-1]

    return trajectory_start_idx, trajectory_end_idx

def _determine_split(session_name, ego_id, start, stop) -> DatasetMode:
    split_frac = 0.8
    start, stop = int(round(start, 0)), int(round(stop, 0))
    hash_num = int(hashlib.sha1(f'{session_name},{ego_id},{start},{stop}'.encode('utf-8')).hexdigest(), 16)
    if (hash_num % 100) / 100 < split_frac:
        return DatasetMode.TRAIN
    else:
        return DatasetMode.VALIDATION

class OpenDDRecording():

    def __init__(self, *, dataset: OpenDDDataset, timedelta_s: float = DT,
                 dataset_mode: DatasetMode = DatasetMode.TRAIN) -> None:
        self._dataset = dataset
        self._dataset_mode = dataset_mode
        self._env_vehicles = {}
        self._df: Optional[pd.DataFrame] = None
        self._frame = 0
        self._timedelta_s = timedelta_s
        self._timestamps = []
        self._session_name: Optional[str] = None

        self._transformer: Optional[Utm2CarlaMapper] = None

    def reset(self, session_name, seed=None):
        if self._df is not None:
            del self._df

        self._session_name = session_name
        with sqlite3.connect(self._dataset.db_path) as conn:
            df = pd.read_sql(f'select * from {session_name}', conn)
            # for now do not extract pedestrians, bicycles and trailers
            df = df[~df.CLASS.str.contains('Pedestrian|Bicycle|Trailer')]
            df = _resample_df(df, self._timedelta_s)
            self._timestamps = np.arange(df.TIMESTAMP.min(),
                                         df.TIMESTAMP.max() + self._timedelta_s,
                                         self._timedelta_s)
            self._df = df

        # search for train/validation roundabout pass
        dataset_mode = None
        if seed is not None:
            random.seed(seed)
        while dataset_mode != self._dataset_mode:
            ego_id, timestamp_start_s, timestamp_end_s = _find_ego_vehicle_with_time_frame(self.place, self._df)
            dataset_mode = _determine_split(session_name, ego_id, timestamp_start_s, timestamp_end_s)

        self._frame = np.where(np.isclose(self._timestamps, timestamp_start_s, 0.0001))[0][0] + 1
        self._env_vehicles = {}

        self._transformer = Utm2CarlaMapper(self.place)
        return ego_id, timestamp_start_s, timestamp_end_s

    def step(self) -> List[RealTrafficVehicle]:
        timestamp_s = self._timestamps[self._frame]
        vehicles_current_ids = self._df[
            np.isclose(self._df.TIMESTAMP, timestamp_s)
        ].OBJID.to_list()

        for vehicle_id in vehicles_current_ids:
            if vehicle_id not in self._env_vehicles:
                # TODO: check if x/y smoothing is not required (in ngsim dataset there is smoothing in 15 frames wnd)
                new_vehicle_df = self._df[
                    (self._df.OBJID == vehicle_id) &
                    ((self._df.TIMESTAMP >= timestamp_s) | np.isclose(self._df.TIMESTAMP, timestamp_s))
                ]
                self._env_vehicles[vehicle_id] = OpenDDVehicle(new_vehicle_df, self._transformer)

        self._env_vehicles = {k: v for k, v in self._env_vehicles.items() if not v.has_finished}

        real_traffic_vehicles = [v.as_real_traffic_car() for v in self._env_vehicles.values()]
        if real_traffic_vehicles:
            if len(real_traffic_vehicles) > 1:
                assert all([
                    np.isclose(v1.timestamp_s, v2.timestamp_s)
                    for v1, v2 in more_itertools.windowed(real_traffic_vehicles, 2)
                ]), (
                    self._session_name,
                    [v.debug for v in real_traffic_vehicles],
                )
            assert np.isclose(real_traffic_vehicles[0].timestamp_s, timestamp_s), \
                (real_traffic_vehicles[0].timestamp_s, timestamp_s)

        self._frame += 1
        for v in self._env_vehicles.values():
            v.step()

        return real_traffic_vehicles

    def close(self):
        pass

    @property
    def place(self) -> Place:
        place_name = self._session_name.split('_')[0]
        return self._dataset.places[place_name]

    @property
    def session_name(self) -> str:
        return self._session_name

    @property
    def timestamp_s(self) -> float:
        return self._timestamps[self._frame]

    @property
    def transformer(self):
        return self._transformer

    def get_df_by_objid(self, ego_id):
        return self._df[self._df.OBJID == ego_id]

    @property
    def has_finished(self):
        return self._frame >= len(self._timestamps) - 1


from django.conf.urls import url

from .views import *

urlpatterns = [
    # 课程列表
    url(r'^list/$', CourseListView.as_view(), name='list'),
    # 课程详情
    url(r'^detail/(?P<course_id>\d+)/$', DetailView.as_view(), name='detail'),
    # 视频信息
    url(r'^info/(?P<course_id>\d+)/$', InfoView.as_view(), name='info'),
    # 课程评论
    url(r'^comment/(?P<course_id>\d+)/$', CommentView.as_view(), name='comment'),
    # 添加评论
    url(r'^addcomment/$', AddComment.as_view(), name='addcomment'),
    #video
    url(r'^video/(?P<video_id>\d+)/$', VideoView.as_view(), name='video'),
]


import os
from celery import Celery
from celery.schedules import crontab

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings')

app = Celery('backend')
app.config_from_object('django.conf:settings', namespace='CELERY')

app.conf.beat_schedule = {
   'check_mail_everyday': {
       'task': 'emailService.task.check_dates_task',
       'schedule': crontab(hour=1, minute=00),

   }
}

app.autodiscover_tasks()


from functools import wraps

def vowel_filter(function):
    vowels = ["a", "e", "i", "o", "u", "y"]
    found_vowels = []

    @wraps(function)
    def wrapper():
        for ch in function():
            if ch.lower() in vowels:
                found_vowels.append(ch)

        return found_vowels

    return wrapper

@vowel_filter
def get_letters():
    return ["a", "b", "c", "d", "e"]

print(get_letters())


from django.shortcuts import render, redirect
from django.utils import timezone
from .forms import ActivateSertificateForm

from programs.models import Category
from .models import Sertificate

# Create your views here.
def activate_sertificate(request):
    if request.method == "POST":
        form = ActivateSertificateForm(request.POST)
        if form.is_valid():
            sert = form.save(commit=False)
            try:
                sertificate = Sertificate.objects.get(number = sert.number)
                if not sertificate.is_active:
                    sertificate.user = request.user
                    sertificate.activation_date = timezone.now()
                    sertificate.is_active = True
                    sertificate.save()
                    return redirect('success/')
                else:
                    return redirect('outdated/')
            except Exception as e:
                return redirect('unsuccess/')

    else:
        form = ActivateSertificateForm()
        categories = Category.objects.all();
    return render(request, 'activation/check_sertificate.html', {'form': form, 'categories': categories })

def return_success_message(request):
    categories = Category.objects.all();
    return render(request, 'activation/success.html', { 'categories': categories })

def return_unsuccess_message(request):
    categories = Category.objects.all();
    return render(request, 'activation/unsuccess.html', { 'categories': categories })

def return_outdated_message(request):
    categories = Category.objects.all();
    return render(request, 'activation/outdated.html', { 'categories': categories })


from gensim.models.doc2vec import Doc2Vec
import pickle

def get_most_similar_docs(test_data, model_path):
   # Load the Doc2Vec model
   model = Doc2Vec.load(model_path)

   # Split the test_data string into a list of words
   test_data_words = test_data.split()

   # Infer the vector for the test document
   inferred_vector = model.infer_vector(test_data_words)

   with open('./homework_6/data/tagged_data.pkl', 'rb') as file:
        loaded_tagged_data = pickle.load(file)
   # Get the 5 most similar documents based on the inferred vector
   sims = model.dv.most_similar([inferred_vector], topn=5)
   idx = [sims[i][0] for i in range(5)]

   # Print the most similar documents
   print('Test Document: «{}»\n'.format(' '.join(test_data_words)))
   print(u'SIMILAR DOCS PER MODEL %s:\n' % model)
   for label, index in [('1', 0), ('2', 1), ('3', 2), ('4', 3), ('5', 4)]:
       print(u'%s %s: «%s»\n' % (label, sims[index], ' '.join(loaded_tagged_data[int(sims[index][0])].words)))
   return idx

test_data = 'exotic yellow spice note meet lean lime pith light crisp nose old vine expression historic winery . meyer lemon rind juice show brightly palate grippy chalkiness complement rich lemon curd flavor'
get_most_similar_docs(test_data, './homework_6/models/doc2vec.model')


import os

from flask import Flask
from flask_jwt_extended import JWTManager
from flask_login import LoginManager

from .auth import ldap_handler
from .db import database
from .db.models import *
from .messages import messages
from .mocks import fake_ldap_handler

configuration_switch = {
    "default": "backend.config.DevConfig",  # Development configuration (fake LDAP)
    "staging": "backend.config.StagingConfig",  # Staging configuration (should be as close as possible to prod)
    "production": "backend.config.ProductionConfig",  # Production configuration
}

ENV = os.environ.get("ENV", "default")

# SET UP =====================================

LOGIN_MANAGER = LoginManager()
LOGIN_MANAGER.login_view = "auth.login"
LOGIN_MANAGER.login_message = messages.LOGIN_MANAGER_MESSAGE

LDAP = fake_ldap_handler.FAKE_LDAP if ENV == "default" else ldap_handler.LDAP
DB = database.DB
JWT_MANAGER = JWTManager()

# ===================================================

def create_app(test_configuration=None, test_db=None):
    """Application factory method"""
    app = Flask(__name__, static_folder="../build/static", template_folder="../build")

    # Configure the application
    if test_configuration:
        app.config.update(test_configuration)
    else:
        app.config.from_object(configuration_switch[ENV])

    # Register extensions ################################
    # |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  #
    ######################################################
    LDAP.init_app(app)
    LOGIN_MANAGER.init_app(app)
    JWT_MANAGER.init_app(app)

    if test_configuration:
        test_db.init_app(app)
    else:
        DB.init_app(app)

    ############################################################

    from .views import auth, index, home, configuration, plugin

    app.register_blueprint(auth.bp)
    app.register_blueprint(index.bp)
    app.register_blueprint(home.bp)
    app.register_blueprint(configuration.bp)
    app.register_blueprint(plugin.bp)

    return app


import time
from functools import wraps

from MatrixDecomposition import MatrixDecomposition
from MatrixGeneration import MatrixGeneration

def fn_timer(function):
    @wraps(function)
    def function_timer(*args, **kwargs):
        t0 = time.time()
        result = function(*args, **kwargs)
        t1 = time.time()
        print("Total time running '%s': %s seconds" % (function.__name__, str(t1 - t0)))
        return result

    return function_timer

class Analyzer:

    @staticmethod
    def analyze_tridiagonal():
        Analyzer.analyze_gauss(10, MatrixGeneration.tridiagonal)
        Analyzer.analyze_seidel(10, MatrixGeneration.tridiagonal)
        Analyzer.analyze_gauss(50, MatrixGeneration.tridiagonal)
        Analyzer.analyze_seidel(50, MatrixGeneration.tridiagonal)
        Analyzer.analyze_gauss(100, MatrixGeneration.tridiagonal)
        Analyzer.analyze_seidel(100, MatrixGeneration.tridiagonal)

    @staticmethod
    def analyze_hilbert():
        Analyzer.analyze_gauss(10, MatrixGeneration.hilbert)
        Analyzer.analyze_seidel(10, MatrixGeneration.hilbert)
        Analyzer.analyze_gauss(50, MatrixGeneration.hilbert)
        Analyzer.analyze_seidel(50, MatrixGeneration.hilbert)
        Analyzer.analyze_gauss(100, MatrixGeneration.hilbert)
        Analyzer.analyze_seidel(100, MatrixGeneration.hilbert)

    @staticmethod
    @fn_timer
    def analyze_gauss(n, method):
        print(f'\'{method.__name__}\' {n}')
        matrix = method(n)
        right = MatrixGeneration.right(matrix)
        matrix_decomposition = MatrixDecomposition(matrix)
        gauss = matrix_decomposition.solve_by_gauss(right)

    @staticmethod
    @fn_timer
    def analyze_seidel(n, method):
        print(f'\'{method.__name__}\' {n}')
        matrix = method(n)
        right = MatrixGeneration.right(matrix)
        matrix_decomposition = MatrixDecomposition(matrix)
        seidel = matrix_decomposition.solve_by_seidel(right, 1e-3)


from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField
from wtforms.validators import DataRequired, Length, ValidationError
from urllib.parse import urlparse
from app.models import Business

def business_name_exists(form, field):
    business_name = field.data
    business = Business.query.filter(Business.name == business_name).first()
    if business:
        raise ValidationError("Business with this name already exists.")

def valid_zip_code(form, field):
    if not field.data.isdigit() or len(field.data) != 5:
        raise ValidationError("Invalid ZIP Code.")

def valid_phone_number(form, field):
    if not field.data.isdigit() or len(field.data) != 10:
        raise ValidationError("Invalid phone number.")

# def valid_url(form, field):
#     try:
#         result = urlparse(field.data)
#         if not all([result.scheme, result.netloc]):
#             raise ValueError()
#     except ValueError:
#         raise ValidationError("Invalid URL.")

class BusinessForm(FlaskForm):
    # class Meta:
    #     csrf = False
    name = StringField('Business Name', validators=[
        Length(min=1, max=50),
        DataRequired(),
        # business_name_exists
    ])

    address = StringField('Address', validators=[
        Length(min=1, max=255),
        DataRequired()
    ])

    city = StringField('City', validators=[
        Length(min=1, max=50),
        DataRequired()
    ])

    state = StringField('State', validators=[
        Length(min=1, max=25),
        DataRequired()
    ])

    zip_code = StringField('Zip Code', validators=[
        Length(min=1, max=10),
        DataRequired(),
        # valid_zip_code
    ])

    phone_number = StringField('Phone Number', validators=[
        Length(min=1, max=30),
        DataRequired(),
        # valid_phone_number
    ])

    category_id = IntegerField('Category ID', validators=[DataRequired()])

    owner_id = IntegerField('Owner ID', validators=[DataRequired()])

    website = StringField('Website', validators=[
        Length(min=1, max=255),
        DataRequired(),
        # valid_url
    ])

    about = StringField('About', validators=[
        Length(min=1, max=500),
        DataRequired()
    ])

    type = StringField('Type', validators=[
        Length(min=1, max=255),
        DataRequired()
    ])


from telegram import Update
from telegram.ext import (
    Updater,
    CallbackContext,
    run_async,
    CommandHandler,
)
from utils import Config
from pkgutil import walk_packages
from types import ModuleType
from typing import Dict

from utils import get_filter

submodules: Dict[str, ModuleType] = {
    module_name: loader.find_module(module_name).load_module(module_name)
    for loader, module_name, is_package in walk_packages(__path__)
}

def describe():
    return "列出所有的指令, 需注意列出的指令在当前的环境内不一定可用"

def run(update: Update, context: CallbackContext):
    update.effective_message.reply_text(
        text="所有指令如下:\n"
        + "\n".join(
            [f"/{command}: {description}"
             for command, description in commands_list]
        ),
        quote=True
    )

commands_list = tuple(
    (name, module.describe())
    for name, module in submodules.items()
) + (
    (__name__, describe()),
)

def register(updater: Updater):
    for module in submodules.values():
        module.register(updater)

    dp = updater.dispatcher
    dp.add_handler(CommandHandler(
        __name__, run, filters=get_filter(Config.watchers), run_async=True))
    # dp.add_handler(CommandHandler(__name__, run, filters=Filters.all)) # DEBUG

    # * Unavailable until all commands are implemented (or at least their describe methods return a string with len > 3)
    updater.bot.set_my_commands(commands_list)


import os
import numpy as np
import torch
import transforms3d
def plane2pose(plane_parameters):
    r3 = plane_parameters[:3]
    r2 = np.zeros_like(r3)
    r2[0], r2[1], r2[2] = (-r3[1], r3[0], 0) if r3[2] * r3[2] <= 0.5 else (-r3[2], 0, r3[0])
    r1 = np.cross(r2, r3)
    pose = np.zeros([4, 4], dtype=np.float32)
    pose[0, :3] = r1
    pose[1, :3] = r2
    pose[2, :3] = r3
    pose[2, 3] = plane_parameters[3]
    pose[3, 3] = 1
    return pose
def plane2euler(plane_parameters, axes='sxyz'):
    pose = plane2pose(plane_parameters)
    T, R, Z, S = transforms3d.affines.decompose(pose)
    euler = transforms3d.euler.mat2euler(R, axes=axes)
    return T, euler


from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.urls import reverse_lazy
from django.shortcuts import render
from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView
from .models import *
from .filters import PostFilter
from datetime import datetime
from .forms import PostForm

class PostList(ListView):
    model = Post
    ordering = 'title'
    template_name = 'news.html'
    context_object_name = 'news'
    paginate_by = 10

    # def get_queryset(self):
    #     queryset = super().get_queryset()
    #     self.filterset = PostFilter(self.request.GET, queryset)
    #     return self.filterset.qs
    #
    # def get_context_data(self, **kwargs):
    #     context = super().get_context_data(**kwargs)
    #     # context['time_now'] = datetime.utcnow()
    #     # context['next_sale'] = None
    #     # context['sorted_posts'] = Post.objects.filter().order_by('-dateCreation')
    #     context['filterset'] = self.filterset
    #     return context

class PostDetail(DetailView):
    model = Post
    template_name = 'post.html'
    context_object_name = 'post'

class PostSearch(ListView):
    model = Post
    ordering = ['dateCreation']
    template_name = 'search.html'
    context_object_name = 'news'
    paginate_by = 10
    form_class = PostForm

    def get_queryset(self):
        queryset = super().get_queryset()
        self.filterset = PostFilter(self.request.GET, queryset)
        return self.filterset.qs

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['filterset'] = self.filterset
        # context['']
        return context

# class PostSearch(ListView):
#     model = Post
#     template_name = "search.html"
#     context_object_name = "news"
#     ordering = ["dateCreation"]
#     paginate_by = 10
#     form_class = PostForm
#
#     def get_filter(self):
#         return PostFilter(self.request.GET, queryset=super().get_queryset())
#
#     def get_queryset(self):
#         return self.get_filter().qs
#
#     def get_context_data(self, *args, **kwargs):
#         return {
#             **super().get_context_data(*args, **kwargs),
#             "filterset": self.get_filter(),
#         }

# def create_post(request):
#     if request.method == 'POST':
#         form = PostForm(request.POST)
#         form.save()
#         return HttpResponseRedirect('/news/')
#     form = PostForm()
#     return render(request, 'post_edit.html', {'form':form})

class PostCreate(PermissionRequiredMixin, CreateView):
    permission_required = ('news.add_post',)
    raise_exception = True
    form_class = PostForm
    model = Post
    template_name = 'post_edit.html'

    # def form_valid(self, form):
    #     product = form.save(commit=False)
    #     post.categoryType.choices = 'NW'
    #     return super().form_valid(form)

class PostUpdate(PermissionRequiredMixin, UpdateView):
    permission_required = ('news.change_post',)
    form_class = PostForm
    model = Post
    template_name = 'post_edit.html'

class PostDelete(PermissionRequiredMixin, DeleteView):
    permission_required = ('news.delete_post',)
    model = Post
    template_name='post_delete.html'
    success_url= reverse_lazy('post_list')


import numpy as np
import netCDF4 as nc4
import net_radiation
import atmospheric_parameters
import wind_shear_velocity
import datetime

# Using TerraClimate
# 2.5 arcminute (1/24 degree) resolution: ~5 km N-S

# Import step
# ... load files here or with a CLI
years = range(1958, 2019)
months_zero_indexed = range(12)
TerraClimateDir = '/media/andy/data1/TerraClimate/'

def extract_data(varnc, varname, varmonth_zero_indexed=None):
    if varmonth_zero_indexed is None:
        var = varnc.variables[varname][:]
    else:
        var = varnc.variables[varname][varmonth_zero_indexed]
    fv = var.fill_value
    var = var.data
    var[var == fv] = np.nan
    return var

# Get lats and lons from one file
srad_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_srad_1958.nc')
lats = extract_data(srad_nc, 'lat')
lons = extract_data(srad_nc, 'lon')
#LONS, LATS = np.meshgrid (lons, lats)

# Shear velocity of winds: tool to compute from velocity
ustar_interp = wind_shear_velocity.create_lookup_table_one_step()

# Elevation
elevation_nc = nc4.Dataset(TerraClimateDir+'Gebco_2020_2_point_5_arcminute.nc')
elevation = extract_data(elevation_nc, 'value')
elevation = elevation[::-1]

# Heat capacity of air
specific_heat_capacity_of_air = 1.005 # approx. constant at 1 atm
                                      # Humidity minor impact below 40C or so
                                      # But this is an approximation!
cp = specific_heat_capacity_of_air # Easier

# Water density
rho_w = 1000.

# Latent heat of vaporization for water
Lv = 2.5E6
DeltaH_vap = Lv # to make me happier

# Ratio of molecular weight of water vapor to dry air
epsilon = 0.622

# Days in month, for weighting
days_in_month = [31, 28.25, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

# Evaporation array
E = np.zeros(elevation.shape)

years = years[:13] # 1970 and before: before so much GW
for year in years:
    print(year)
    # Incoming solar radiation (monthly average)
    srad_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_srad_'+str(year)+'.nc')
    # Maximum daily temperature (monthly average)
    tmax_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_tmax_'+str(year)+'.nc')
    # Minimum daily temperature (monthly average)
    tmin_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_tmin_'+str(year)+'.nc')
    # Wind speed (monthly average)
    ws_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_ws_'+str(year)+'.nc')
    # Vapor pressure (monthly average)
    vap_nc = nc4.Dataset(TerraClimateDir+'TerraClimate_vap_'+str(year)+'.nc')

    # Now compute for each month
    for month_zi in months_zero_indexed:
        print(month_zi+1)
        # Data
        srad = extract_data(srad_nc, 'srad', month_zi)
        tmax = extract_data(tmax_nc, 'tmax', month_zi)
        tmin = extract_data(tmin_nc, 'tmin', month_zi)
        ws = extract_data(ws_nc, 'ws', month_zi)
        vap = extract_data(vap_nc, 'vap', month_zi) * 1000. # kPa to Pa
        # Average radiation on the midday of the month; could be more precise
        date = datetime.date(year, month_zi+1, int(np.round(days_in_month[month_zi]/2.)))
        #elevation = 2000. # placeholder
        #julian_day = 205 # placeholder
        #vap = .03*101325 # placeholder
        albedo = 0.06

        # Calculations:
        # Net Radiation
        Rn = net_radiation.computeNetRadiation(elevation, date, lats, len(lons),
                                                tmax, tmin, vap, srad, albedo)

        # Shear velocity of winds
        ustar = ustar_interp(ws)

        # Vapor-pressure deficit
        # We don't have max and min humidity
        VPD = atmospheric_parameters.compute_vpd( (tmax+tmin)/2., vap )

        # Atmospheric pressure
        P = atmospheric_parameters.compute_atmospheric_pressure(elevation)

        # Atmospheric density (ignoring temperature + humidity effects)
        rho_a = atmospheric_parameters.compute_atmospheric_density(elevation,
                                        (tmax + tmin)/2.)

        # Clausius-Clayperon phase-change slope
        Delta = ( atmospheric_parameters.compute_Delta_e_sat( tmax )
                  + atmospheric_parameters.compute_Delta_e_sat( tmin ) ) / 2.

        _E = (Rn + cp*rho_a*ustar**2/(Delta*ws) * VPD) \
             / ( rho_w*Lv  + P*cp*rho_w/epsilon )
        _E[_E<0] = 0 # ignore condensation; I think it's spurious (Antarctica?)
        E += _E*days_in_month[month_zi]

E /= (365.25*len(years))

# Export
from netCDF4 import Dataset
import numpy
import time

#path to the file you want to open or create
location_string="evaporation_002p5_arcmin.nc"

# Create nc file
rootgrp = Dataset(location_string,"w",format="NETCDF4")

# Dimensions
lat=rootgrp.createDimension("lat", len(lats))
lon=rootgrp.createDimension("lon", len(lons))
value = rootgrp.createDimension("evaporation", None)

# Values
latitudes = rootgrp.createVariable("lat", "f4", ("lat",))
longitudes = rootgrp.createVariable("lon", "f4", ("lon",))
values = rootgrp.createVariable("value", "f4" , ("lat", "lon",))
latitudes[:] = lats
longitudes[:] = lons
values[:] = E

# Units
latitudes.units = "degrees north"
longitudes.units = "degrees east"
values.units = "metres per second"

# Metadata
rootgrp.description = "Evaporation derived from TerraClimate data products (see https://github.com/umn-earth-surface/TerraClimate-potential-open-water-evaporation)."
rootgrp.history = "created" + time.ctime(time.time())
values.Long_Name = "Open-water evaporation"

# Save
rootgrp.close()


import openpyxl as xl
import xlwings as xw
from Worksheet import Worksheet,QPreviewItem
from Workcell import *
from PyQt4.QtGui import *
from PyQt4.QtCore import *
from copy import copy
#import time
import datetime
##################################################
#       class for PS sheet handling
##################################################
UP = 0
DOWN = 1
nn = 0
class PSsheet(Worksheet):
    def __init__(self,sheet = None,sheet_wr = None):
        super(PSsheet,self).__init__(sheet,sheet_wr)
        self._preview_model_list = []
        self._extended_preview_model_list = None
        self.init_ps_sheet()
        self.init_ps_model()
    def __del__(self):
        super(PSsheet,self).__del__()
        del self._preview_model
        del self._extended_preview_model_list

    def init_ps_sheet(self):
        if self._xmlname != None:
            self._status = self.search_header_by_value(u'Status(POR,INIT,PREV)')
            self._subject_matter = self.search_header_by_value(u'Subject Matter/\nFunctional Area')
            self._container_name = self.search_header_by_value(u'Container Name\nTechnical Specification')
    def init_ps_model(self):
        self._preview_model = QStandardItemModel()
        self._preview_model.setColumnCount(4)
        self._preview_model.setHeaderData(0,Qt.Horizontal,'status')
        self._preview_model.setHeaderData(1,Qt.Horizontal,'subject matter')
        self._preview_model.setHeaderData(2,Qt.Horizontal,'container name')
        self._preview_model.setHeaderData(3,Qt.Horizontal,'xmlname')
        self._extended_preview_model = None
    def update_model(self):
        super(PSsheet,self).update_model()
        self._preview_model_list = []
        self.init_ps_model()
        self.init_ps_sheet()
        if self._xmlname != None:
            try:
                for xml_name in self.xml_names():
                    item_status = QPreviewItem(self._status.get_item_by_xmlname(xml_name))
                    item_subject_matter = QPreviewItem(self._subject_matter.get_item_by_xmlname(xml_name))
                    item_container_name = QPreviewItem(self._container_name.get_item_by_xmlname(xml_name))
                    item_xml_name = QPreviewItem(xml_name)
                    self._preview_model.appendRow((item_status,item_subject_matter,item_container_name,item_xml_name))
                    self._preview_model_list.append((item_status.value,item_subject_matter.value,item_container_name.value,item_xml_name.value))
            except:
                return 'error'
    def status(self):
        cells = list(self._worksheet.iter_cols(min_col=self._status.col,min_row=self._status.row+1,max_col=self._status.col,max_row=self.max_row).next())
        while cells[-1].value == None:
            cells.pop()
        return map(lambda x:Status(x,self._worksheet_wr),cells)
    def cell(self,row,col):
        #return self._worksheet_wr.range(row,col)
        return self._worksheet.cell(row=row,column=col)

    def auto_fit(self,cols):
        for col in cols:
            for i in range(1):
                self._worksheet_wr.api.Columns(col).AutoFit()

    def add_row(self,start_pos,offset,orientation):
        loop = offset
        while loop > 0:
            self._worksheet_wr.api.Rows[start_pos].Insert(-4121)
            loop -= 1

    def delete_row(self,start_pos,offset):
        self._worksheet_wr.range('%d:%d'%(start_pos,start_pos+offset-1)).api.Delete()
    def lock_row(self,row,status):
        self._worksheet_wr.api.Rows[row-1].Locked = True
    def lock_sheet(self):
        self._worksheet_wr.api.Protect()
    def lock_sheet_status(self):
        return self._worksheet_wr.api.ProtectContents
    def unlock_sheet(self):
        self._worksheet_wr.api.Unprotect()
    def unlock_all_cells(self):
        self._worksheet_wr.api.Cells.Locked = False

    def extended_preview_model(self):
        if self._extended_preview_model == None:
            self._extended_preview_model = QStandardItemModel()
            self._extended_preview_model.setColumnCount(self._worksheet.max_column)
            for row in self._worksheet.rows:
                item_row = []
                for cell in row:
                    try:
                        if cell.value == None:
                            item = QStandardItem('')
                        else:
                            item = QStandardItem(cell.value)
                    except:
                        item = QStandardItem('')
                    item_row.append(item)
                self._extended_preview_model.appendRow(item_row)
        return self._extended_preview_model
    @property
    def extended_preview_model_list(self):
        if self._extended_preview_model_list == None:
            self._extended_preview_model_list = []
            for row in self._worksheet.iter_rows(min_row=self.min_row,max_row=self.max_row,min_col=self.min_col,max_col=self.max_col):
                item_row = []
                for cell in row:
                    #item_row.append(cell.value if cell.value is not None else '')
                    try:
                        #item_row.append('' if cell.value is None else str(cell.value) if type(cell.value) == long else '%s-%s-%s'%((cell+datetime.delta(days=1)).timetuple().tm_year,(cell+datetime.delta(days=1)).timetuple().tm_mon,(cell+datetime.delta(days=1)).timetuple().tm_mday) if type(cell) is datetime.datetime else (cell.value))
                        item_row.append('' if cell.value is None else str('%s-%s-%s'%((cell.value+datetime.timedelta(days=1)).timetuple().tm_year,(cell.value+datetime.timedelta(days=1)).timetuple().tm_mon,(cell.value+datetime.timedelta(days=1)).timetuple().tm_mday)) if type(cell.value) is datetime.datetime else str(cell.value))
                    except:
                        item_row.append('')
                self._extended_preview_model_list.append(item_row)
        return self._extended_preview_model_list

    @property
    def preview_model(self):
        return self._preview_model_list


#!/usr/bin/python3
import subprocess
import json
import requests
import time
import logging
import os

#bin Paths
ipfspath = '/usr/local/bin/ipfs'
wgetpath = '/usr/bin/wget'
wcpath = '/usr/bin/wc'

#Basic logging to ipfspodcastnode.log
logging.basicConfig(format="%(asctime)s : %(message)s", datefmt="%Y-%m-%d %H:%M:%S", filename="ipfspodcastnode.log", filemode="w", level=logging.INFO)

#Create an empty email.cfg (if it doesn't exist)
if not os.path.exists('cfg/email.cfg'):
  with open('cfg/email.cfg', 'w') as ecf:
    ecf.write('')

#Init IPFS (if necessary)
if not os.path.exists('ipfs/config'):
  logging.info('Initializing IPFS')
  ipfs_init = subprocess.run(ipfspath + ' init', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

#Start WebUI
import webui
logging.info('Starting Web UI')

#Automatically discover relays and advertise relay addresses when behind NAT.
swarmnat = subprocess.run(ipfspath + ' config --json Swarm.RelayClient.Enabled true', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

#Start IPFS
daemon = subprocess.run(ipfspath + ' daemon >/dev/null 2>&1 &', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logging.info('Starting IPFS Daemon')
time.sleep(10)

#Get IPFS ID
with open('ipfs/config', 'r') as ipcfg:
  ipconfig = ipcfg.read()
  jtxt = json.loads(ipconfig)
  logging.info('IPFS ID : ' + jtxt['Identity']['PeerID'])

#Main loop
while True:

  #Request payload
  payload = { 'version': 0.6, 'ipfs_id': jtxt['Identity']['PeerID'] }

  #Read E-mail Config
  with open('cfg/email.cfg', 'r') as ecf:
    email = ecf.read()
    if email == '':
      email = 'user@example.com'
  payload['email'] = email

  #Check if IPFS is running, restart if necessary.
  payload['online'] = False
  diag = subprocess.run(ipfspath + ' diag sys', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  if diag.returncode == 0:
    ipfs = json.loads(diag.stdout)
    payload['ipfs_ver'] = ipfs['ipfs_version']
    payload['online'] = ipfs['net']['online']
  if payload['online'] == False:
    #Start the IPFS daemon
    daemon = subprocess.run(ipfspath + ' daemon >/dev/null 2>&1 &', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    logging.info('@@@ IPFS NOT RUNNING !!! Restarting Daemon @@@')

  #Get Peer Count
  peercnt = 0
  speers = subprocess.run(ipfspath + ' swarm peers|wc -l', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  if speers.returncode == 0:
    peercnt = speers.stdout.decode().strip()
  payload['peers'] = peercnt

  #Request work
  logging.info('Requesting Work...')
  try:
    response = requests.post("https://IPFSPodcasting.net/Request", timeout=120, data=payload)
    work = json.loads(response.text)
    logging.info('Response : ' + str(work))
  except requests.RequestException as e:
    logging.info('Error during request : ' + str(e))
    work = { 'message': 'Request Error' }

  if work['message'] == 'Request Error':
    logging.info('Error requesting work from IPFSPodcasting.net (check internet / firewall / router).')

  elif work['message'][0:7] != 'No Work':
    if work['download'] != '' and work['filename'] != '':
      logging.info('Downloading ' + str(work['download']))
      #Download any "downloads" and Add to IPFS (1hr48min timeout)
      try:
        hash = subprocess.run(wgetpath + ' -q --no-check-certificate "' + work['download'] + '" -O - | ' + ipfspath + ' add -q -w --stdin-name "' + work['filename'] + '"', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=6500)
        hashcode = hash.returncode
      except subprocess.SubprocessError as e:
        logging.info('Error downloading/pinning episode : ' + str(e))
        hashcode = 99

      if hashcode == 0:
        #Get file size (for validation)
        downhash=hash.stdout.decode().strip().split('\n')
        size = subprocess.run(ipfspath + ' cat ' + downhash[0] + ' | ' + wcpath + ' -c', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        downsize=size.stdout.decode().strip()
        logging.info('Added to IPFS ( hash : ' + str(downhash[0]) + ' length : ' + str(downsize) + ')')
        payload['downloaded'] = downhash[0] + '/' + downhash[1]
        payload['length'] = downsize
      else:
        payload['error'] = hashcode

    if work['pin'] != '':
      #Directly pin if already in IPFS
      logging.info('Pinning hash (' + str(work['pin']) + ')')
      try:
        pin = subprocess.run(ipfspath + ' pin add ' + work['pin'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=6500)
        pincode = pin.returncode
      except subprocess.SubprocessError as e:
        logging.info('Error direct pinning : ' + str(e))
        #Clean up any other pin commands that may have spawned
        cleanup = subprocess.run('kill `ps aux|grep "ipfs pin ad[d]"|awk \'{ print $2 }\'`', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        pincode = 98

      if pincode == 0:
        #Verify Success and return full CID & Length
        pinchk = subprocess.run(ipfspath + ' ls ' + work['pin'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if pinchk.returncode == 0:
          hashlen=pinchk.stdout.decode().strip().split(' ')
          payload['pinned'] = hashlen[0] + '/' + work['pin']
          payload['length'] = hashlen[1]
        else:
          payload['error'] = pinchk.returncode
      else:
        payload['error'] = pincode

    if work['delete'] != '':
      #Delete/unpin any expired episodes
      logging.info('Unpinned old/expired hash (' + str(work['delete']) + ')')
      delete = subprocess.run(ipfspath + ' pin rm ' + work['delete'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      payload['deleted'] = work['delete']

    #Report Results
    logging.info('Reporting results...')
    #Get Usage/Available
    repostat = subprocess.run(ipfspath + ' repo stat -s|grep RepoSize', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if repostat.returncode == 0:
      repolen = repostat.stdout.decode().strip().split(':')
      used = int(repolen[1].strip())
    else:
      used = 0
    payload['used'] = used
    df = os.statvfs('/')
    payload['avail'] = df.f_bavail * df.f_frsize

    try:
      response = requests.post("https://IPFSPodcasting.net/Response", timeout=120, data=payload)
    except requests.RequestException as e:
      logging.info('Error sending response : ' + str(e))

  else:
    logging.info('No work.')

  #wait 10 minutes then start again
  logging.info('Sleeping 10 minutes...')
  time.sleep(600)