import re

TITLE = 'Crackle'

ART = 'art-default.jpg'
ICON = 'icon-default.png'

TYPE_MOVIES = 'movies'
TYPE_TELEVISION = 'television'
TYPE_ORIGINALS = 'originals'
TYPE_COLLECTIONS = 'collections'

GENRE_ALL = 'all'
GENRE_ACTION = 'Action'
GENRE_COMEDY = 'Comedy'
GENRE_CRIME = 'Crime'
GENRE_HORROR = 'Horror'
GENRE_SCI_FI = 'Sci-Fi'
GENRE_THRILLER = 'Thriller'

URL_GEO = 'http://api.crackle.com/Service.svc/geo/country?format=json'
URL_CATEGORIES = 'http://api.crackle.com/Service.svc/browse/%s/full/%s/alpha/%s?format=json'
URL_DETAILS = 'http://api.crackle.com/Service.svc/channel/%s/folders/%s?format=json'
URL_MEDIA_DETAILS = 'http://api.crackle.com/Service.svc/details/media/%s/%s?format=json'

###################################################################################################

def Start():

  # Set the types of view groups
  Plugin.AddViewGroup('List', viewMode = 'List', mediaType = 'items')
  Plugin.AddViewGroup('InfoList', viewMode = 'InfoList', mediaType = 'items')

  # Set the default ObjectContainer attributes
  ObjectContainer.title1 = TITLE
  ObjectContainer.view_group = 'List'
  ObjectContainer.art = R(ART)

  # Default icons for DirectoryObject and VideoClipObject in case there isn't an image
  DirectoryObject.thumb = R(ICON)
  DirectoryObject.art = R(ART)
  VideoClipObject.thumb = R(ICON)
  VideoClipObject.art = R(ART)

  # Set the default cache time
  HTTP.CacheTime = CACHE_1DAY

###################################################################################################

@handler('/video/crackle', TITLE, thumb=ICON, art=ART)
def MainMenu():

  # Determine the location of the client, and the associated API region code
  location_details = JSON.ObjectFromURL(URL_GEO)
  if location_details['status']['messageCode'] != '0':
    return MessageContainer(location_details['status']['message'])
  location = location_details['CountryCode']

  oc = ObjectContainer()

  oc.add(DirectoryObject(key = Callback(Genres, title = 'Movies', type = TYPE_MOVIES, location = location), title = 'Movies'))
  oc.add(DirectoryObject(key = Callback(Genres, title = 'Television', type = TYPE_TELEVISION, location = location), title = 'Television'))
  oc.add(DirectoryObject(key = Callback(Genres, title = 'Original', type = TYPE_ORIGINALS, location = location), title = 'Original'))

  # This is currently disabled as it appears that they're still working on it. I originally didn't get any titles, now I get titles
  # which have no associated information.
  # oc.add(DirectoryObject(key = Callback(Genres, title = 'Collections', type = TYPE_COLLECTIONS, location = location), title = 'Collections'))

  return oc

###################################################################################################

def Genres(title, type, location):
  oc = ObjectContainer(title2 = title)

  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'All Genres', type = type, genre = GENRE_ALL, location = location), title = 'All Genres'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Action', type = type, genre = GENRE_ACTION, location = location), title = 'Action'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Comedy', type = type, genre = GENRE_COMEDY, location = location), title = 'Comedy'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Crime', type = type, genre = GENRE_CRIME, location = location), title = 'Crime'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Thriller', type = type, genre = GENRE_THRILLER, location = location), title = 'Thriller'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Horror', type = type, genre = GENRE_HORROR, location = location), title = 'Horror'))
  oc.add(DirectoryObject(key = Callback(ListChannels, title = 'Sci-Fi', type = type, genre = GENRE_SCI_FI, location = location), title = 'Sci-Fi'))

  return oc

###################################################################################################

def ListChannels(title, type, genre, location):
  oc = ObjectContainer(title2 = title, view_group = 'InfoList')

  titles = JSON.ObjectFromURL(URL_CATEGORIES % (type, genre, location))
  for title in titles['Entries']:

    oc.add(DirectoryObject(
      key = Callback(ListTitles, title = title['Name'], id = title['ID'], location = location),
      title = title['Name'],
      summary = title['Description'],
      thumb = title['ChannelArtTileLarge']))

  if len(oc) == 0:
    return MessageContainer("Error", "No titles were found!")

  return oc

###################################################################################################

def ListTitles(title, id, location):
  oc = ObjectContainer(title2 = title, view_group = 'InfoList')

  titles = JSON.ObjectFromURL(URL_DETAILS % (id, location))

  if len(titles['FolderList']) == 0:
    return MessageContainer("Error", "No titles were found!")

  for playlist in titles['FolderList'][0]['PlaylistList']:
    for title in playlist['MediaList']:

      url = title['XItemId']
      video_title = title['Title']
      summary = title['Description']
      thumb = title['ThumbnailExternal']
      genres = [ genre.strip() for genre in title['Genre'].split(',') ]
      content_rating = title['Rating']
      date = Datetime.ParseDate(title['ReleaseDate'])

      duration_text = title['Duration']
      duration_dict = re.match("((?P<hours>[0-9]+):)?(?P<mins>[0-9]+):(?P<secs>[0-9]+)", duration_text).groupdict()

      hours = 0
      if duration_dict['hours'] != None:
          hours = int(duration_dict['hours'])

      mins = int(duration_dict['mins'])
      secs = int(duration_dict['secs'])
      duration = ((((hours * 60) + mins) * 60) + secs) * 1000

      if title['RootChannel'] == 'Movies':
        oc.add(MovieObject(
          url = url,
          title = video_title,
          summary = summary,
          thumb = thumb,
          genres = genres,
          content_rating = content_rating,
          duration = duration,
          originally_available_at = date))

      elif title['RootChannel'] == 'Television' or title['RootChannel'] == 'Originals':

        show = title['ParentChannelName']
        season = int(title['Season'])
        index = int(title['Episode'])

        oc.add(EpisodeObject(
          url = url,
          show = show,
          title = video_title,
          season = season,
          index = index,
          summary = summary,
          thumb = thumb,
          content_rating = content_rating,
          duration = duration,
          originally_available_at = date))

  return oc


from os.path import dirname
from typing import TYPE_CHECKING

import glm

if TYPE_CHECKING:
	import moderngl

	from main import MinecraftClone

class Shaders:

	def __init__(self, mc: 'MinecraftClone') -> None:
		self.minecraft: 'MinecraftClone' = mc

		self.chunk: 'moderngl.Program' = self.getShader(name="chunk")

		self.setUniforms()

	def setUniforms(self):
		self.chunk['projection'].write(self.minecraft.player.projection)
		self.chunk['model'].write(glm.mat4())

	def update(self):
		self.chunk['view'].write(self.minecraft.player.view)

	def getShader(self, name: str) -> 'moderngl.Program':
		with open(f"{dirname(__file__)}/shaders/{name}.vert") as f:
			vertex = f.read()

		with open(f"{dirname(__file__)}/shaders/{name}.frag") as f:
			fragment = f.read()

		return self.minecraft.context.program(vertex_shader=vertex,fragment_shader=fragment)

if __name__ == '__main__':
	from main import main
	main()


import logging
import time as pytime
from pathlib import Path
from typing import Any, Dict, List

import dolfin as df
import ufl
from dolfin import grad, inner

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
df.set_log_level(df.LogLevel.WARNING)

from multidiffusion.boundary import DirichletBoundary, indexed_boundary_conditions, process_dirichlet
from multidiffusion.fenicsstorage import FenicsStorage
from multidiffusion.interpolator import vectordata_interpolator
from multidiffusion.timekeeper import TimeKeeper
from multidiffusion.utils import assign_mixed_function

def print_progress(t, T, rank=0):
    if rank != 0:
        return
    progress = int(40 * t / T)
    print(
        f"[{'=' * progress}{' ' * (40 - progress)}] {round(progress * 100 / 40, 1)}%",
        end="\r",
        flush=True,
    )

def read_concentration_data(filepath, funcname="cdata") -> list[df.Function]:
    store = FenicsStorage(filepath, "r")
    tvec = store.read_timevector(funcname)
    c = store.read_function(funcname, idx=0)
    C = [df.Function(c.function_space()) for _ in range(tvec.size)]
    for idx in range(len(C)):
        store.read_checkpoint(C[idx], funcname, idx)
    return tvec, C

def read_function_element(filepath, funcname) -> df.FunctionSpace:
    store = FenicsStorage(filepath, "r")
    el = store.read_element(funcname)
    store.close()
    return el

def compartment_form(
    idx_j: int,
    u: ufl.Argument,
    v: ufl.Argument,
    c0: df.Function,
    D: Dict[str, float],
    phi: Dict[str, float],
    alpha: float,
    P: float,
    compartments: List[str],
    dt: float,
) -> df.Form:
    j = compartments[idx_j]
    sj = sum(
        [
            (alpha * P / phi[j]) * (u[idx_i] - u[idx_j])
            for idx_i, i in enumerate(compartments)
            if idx_i != idx_j
        ]
    )
    return (u[idx_j] - c0[idx_j] - dt * sj) * v[idx_j] + dt * inner(
        D[j] * grad(u[idx_j]), grad(v[idx_j])
    )

def multicomp_diffusion_form(
    V: df.FunctionSpace,
    coefficients: Dict[str, Any],
    c0: df.Function,
    compartments: List[str],
    dt: float,
) -> df.Form:
    dx = df.Measure("dx", domain=V.mesh())
    u = df.TrialFunction(V)
    v = df.TestFunction(V)
    D = coefficients["diffusion_coefficient"]
    phi = coefficients["porosity"]
    alpha = coefficients["alpha"]
    P = coefficients["permeability"]
    return (
        sum(
            [
                compartment_form(idx_j, u, v, c0, D, phi, alpha, P, compartments, dt)
                for idx_j, _ in enumerate(compartments)
            ]
        )
        * dx
    )

def get_default_coefficients() -> Dict[str, Any]:
    return {
        "porosity": {
            "ecs": 0.14,
            "pvs": 0.01,
        },
        "diffusion_coefficient": {"ecs": 3.4e-4, "pvs": 3.4e-3},
        "alpha": 5.0,
        "permeability": 8.3e-5,
    }

def multicomp_diffusion(compartments, coefficients, inputfile, outputfile=None):
    if outputfile is None:
        outputfile = Path(inputfile)
    logger.info(f"Reading data from {inputfile}")
    timevec, data = read_concentration_data(inputfile)
    interpolator = vectordata_interpolator(data, timevec)
    c0 = data[0].copy(deepcopy=True)

    domain = c0.function_space().mesh()
    el = read_function_element(inputfile, "cdata")
    element = df.MixedElement([el] * 2)
    V = df.FunctionSpace(domain, element)

    dt = 3600
    T = timevec[-1]
    time = TimeKeeper(dt=dt, endtime=T)

    # Define boundary conditions
    u_interp = {compartment: c0.copy(deepcopy=True) for compartment in compartments}
    phi = coefficients["porosity"]
    phi_tot = sum(phi.values())
    for val in u_interp.values():
        val.vector()[:] = (1.0 / phi_tot) * interpolator(0.0)

    boundary_data = {
        idx: [DirichletBoundary(u_interp[compartment], "everywhere")]
        for idx, compartment in enumerate(compartments)
    }
    boundaries = indexed_boundary_conditions(boundary_data)
    bcs = process_dirichlet(V, domain, boundaries)

    # Define variational problem
    u0 = assign_mixed_function(u_interp, V, compartments)
    F = multicomp_diffusion_form(V, coefficients, u0, compartments, dt)
    a = df.lhs(F)
    l = df.rhs(F)
    A = df.assemble(a)

    u = df.Function(V)
    u.assign(u0)
    storage = FenicsStorage(outputfile, "a")
    storage.write_function(u, "multidiffusion", overwrite=True)

    logger.info("Starting time loop...")
    tic = pytime.time()
    time.reset()
    for ti in time:
        print_progress(float(ti), time.endtime, rank=df.MPI.comm_world.rank)
        for j, uj in u_interp.items():
            uj.vector()[:] = ((1.0 / phi_tot)) * interpolator(float(ti))
        b = df.assemble(l)
        for bc in bcs:
            bc.apply(A, b)
        df.solve(A, u.vector(), b, "gmres", "hypre_amg")
        storage.write_checkpoint(u, "multidiffusion", float(ti))
        u0.assign(u)
    storage.close()
    logger.info("Time loop finished.")
    toc = pytime.time()

    df.MPI.comm_world.barrier()
    if df.MPI.comm_world.rank == 0:
        logger.info(f"Elapsed time in loop: {toc - tic:.2f} seconds.")
    return storage.filepath


import config
import alpaca_trade_api as tradeapi

api = tradeapi.REST(config.API_KEY, config.SECRET_KEY, base_url=config.API_URL)

barsets = api.get_barset(['AAPL', 'MSFT'], 'day')

# loop over the keys in the barsets dictionary
for symbol in barsets:
    print(f"processing symbol {symbol}")

    # loop through each bar for the current symbol in the dictionary
    for bar in barsets[symbol]:
        print(bar.t, bar.o, bar.h, bar.l, bar.c, bar.v)


'''
You need to write a function that checks
whether if the number passed into it is a prime number or not.
'''

#Write your code below this line ðŸ‘‡
def prime_checker(number):
  for i in range(2, int(number ** 0.5) + 1):
    if 0 == number % i:
      print(f"{number}, is not a prime number")
      return
  print(f"{number} is a prime number")

#Write your code above this line ðŸ‘†

#Do NOT change any of the code belowðŸ‘‡
n = int(input("Check this number: "))
prime_checker(number=n)


import sys, argparse

def writeline(args, line, writeType):
    if (writeType == 1) and args.output_unique_file1:
        sys.stdout.write(line)
    elif (writeType == 2) and args.output_unique_file2:
        sys.stdout.write('\t{0}'.format(line))
    elif (writeType == 3) and args.output_duplicate:
        sys.stdout.write('\t\t{0}'.format(line))

def compare_files(args):
    # When -u option is not provided, assume that the two files are sorted
    while (args.file1 != []) and (args.file2 != []):
        if args.file1[0] < args.file2[0]:
            writeline(args, args.file1.pop(0), 1)
        elif args.file1[0] == args.file2[0]:
            writeline(args, args.file1.pop(0), 3)
            args.file2.pop(0)
        else:
            writeline(args, args.file2.pop(0), 2)
    for line in args.file1:
        writeline(args, line, 1)
    for line in args.file2:
        writeline(args, line, 2)

def compare_files_unsorted(args):
    args.file1 = list(args.file1)
    args.file2 = list(args.file2)
    printQueueMap1 = {line : [] for line in args.file1}
    printQueueMap2 = {line : [] for line in args.file2}
    sortedFile1 = sorted(args.file1)
    sortedFile2 = sorted(args.file2)

    while (sortedFile1 != []) and (sortedFile2 != []):
        if sortedFile1[0] < sortedFile2[0]:
            printQueueMap1[sortedFile1.pop(0)].append(1)
        elif sortedFile1[0] == sortedFile2[0]:
            printQueueMap1[sortedFile1.pop(0)].append(3)
            printQueueMap2[sortedFile2.pop(0)].append(None)
        else:
            printQueueMap2[sortedFile2.pop(0)].append(2)
    for line in sortedFile1:
        printQueueMap1[line].append(1)
    for line in sortedFile2:
        printQueueMap2[line].append(2)

    # Print out lines according to the order of the original files
    for line in args.file1:
        printQueue = printQueueMap1[line]
        printType = printQueue.pop(0)
        if (printType == 1) and args.output_unique_file1:
            writeline(args, line, 1)
        if (printType == 3) and args.output_duplicate:
            writeline(args, line, 3)

    for line in args.file2:
        printQueue = printQueueMap2[line]
        printType = printQueue.pop(0)
        if (printType == 2) and args.output_unique_file2:
            writeline(args, line, 2)

def main():
    # Initialize the argument parser; add appropriate attributes to it
    parser = argparse.ArgumentParser(prog='comm',
        description='Select or reject lines common to two files')
    # Add required arguments to the parser
    parser.add_argument("-1", action='store_false', dest='output_unique_file1',
        default=True,
        help="Suppress the output column of lines unique to file1.")
    parser.add_argument("-2", action='store_false', dest='output_unique_file2',
        default=True,
        help="Suppress the output column of lines unique to file2.")
    parser.add_argument("-3", action='store_false', dest='output_duplicate',
        default=True,
        help="Suppress the output column of lines duplicated in file1 and file2.")
    parser.add_argument("-u", "--unsorted",
        action='store_true', default=False,
        help="Sort the two files if they are unsorted.")
    parser.add_argument("file1", type=argparse.FileType('r'))
    parser.add_argument("file2", type=argparse.FileType('r'))
    args = parser.parse_args()

    args.file1 = list(args.file1)
    args.file2 = list(args.file2)

    # Start comparing files and write to stdout according to set options
    if args.unsorted:
        compare_files_unsorted(args)
    else:
        compare_files(args)

if __name__ == "__main__":
    main()


from datetime import datetime
from django.shortcuts import render, reverse, redirect, get_object_or_404
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from .models import Catalogue
from .forms import CatalogueForm, BidForm

def view_all(request):
    """Returns the whole catalogue to browse."""
    catalogue = Catalogue.objects.all()
    newest = Catalogue.objects.last()
    return render(request, 'display-all.html', {"catalogue": catalogue, "newest": newest})

def view_one(request, pk):
    """
    Displays details of a specific item.
    Also handles bid functions.
    """
    display = Catalogue.objects.get(id=pk)
    if not request.user.is_authenticated:
        messages.info(request, 'To view full details please login first.')
        context = {
            "display": display,
        }
        return render(request, 'display-one-closed.html', context)

    else:
        open = Catalogue.objects.filter(start__lte=datetime.now()).filter(finish__gte=datetime.now())
        finish = Catalogue.objects.filter(finish__lte=datetime.now())

        if display in finish:
            # Start payment process or go to auction over view.
            if display.last_bidder == request.user:
                return redirect('payment:billing', id=pk)

            context = {
                "display": display,
            }
            return render(request, 'display-one-closed.html', context)

        elif display in open:

            # Save bid and bidder's ID.
            if request.method == 'POST':
                form = BidForm(request.POST)
                if form.is_valid():
                    display.bid = form.cleaned_data.get('bid')
                    display.last_bidder = request.user
                    display.save()

            else:
                # display current bid
                bid_val = display.bid
                form = BidForm(initial={'bid': bid_val})

            context = {
                "display": display,
                "form": form
            }
            return render(request, 'display-one.html', context)

        else:
            # Display details without bidding details, auction not started.
            context = {
                "display": display,
            }

            return render(request, 'display-one-closed.html', context)

def view_era(request, era):
    """Returns a subset of items by historical era."""
    era_subset = Catalogue.objects.filter(era=era)
    newest = Catalogue.objects.filter(era=era).last()
    context = {
        "era_subset": era_subset,
        "newest": newest
        }
    return render(request, 'display-era.html', context)


import math
import random
import matplotlib.pyplot as plt
import matplotlib.cm as cm

G_PI = math.pi
G_2PI = 2 * math.pi

# redundantly redefine trigonometric funcs. because I can't be bothered typing 'math.' every time
def sin(x, unit="rad"):
    if unit=="deg":
        x = math.radians(x)
    return math.sin(x)

def cos(x, unit="rad"):
    if unit=="deg":
        x = math.radians(x)
    return math.cos(x)

def tan(x, unit="rad"):
    if unit=="deg":
        x = math.radians(x)
    return math.tan(x)

def cot(x, unit="rad"):
    if unit=="deg":
        x = math.radians(x)
    return 1/math.tan(x)

def create_ripples(size, level, smoothing=2, init_wavelength=None, use_numpy=False):
    if type(size) == list:

        # 2D
        if len(size) == 2:
            data = []
            for i in range(size[1]):
                data.append([0] * size[0])

            if not init_wavelength:
                wavelength_x = size[0]
                wavelength_y = size[1]
            else:
                wavelength_x = init_wavelength[0]
                wavelength_y = init_wavelength[1]

            for i in range(level):
                amplitude_x = random.uniform(-1, 1) / (i + 1)**smoothing
                amplitude_y = random.uniform(-1, 1) / (i + 1)**smoothing

                for ex in range(size[0]):
                    for ey in range(size[1]):
                        data[ey][ex] += sin(ex / wavelength_x * G_2PI) * amplitude_x
                        data[ey][ex] += sin(ey / wavelength_y * G_2PI) * amplitude_y

                wavelength_x *= 0.5
                wavelength_y *= 0.5

            plt.imshow(data)
            plt.show()

            if use_numpy:
                import numpy as np
                fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
                X = list(range(size[0]))
                Y = list(range(size[1]))
                X, Y = np.meshgrid(X, Y)
                Z = np.array(data)

                surf = ax.plot_surface(X, Y, Z, cmap=cm.coolwarm)
                plt.show()

        # 3D
        elif len(size) == 3:
            pass

    # 1D
    else:
        data = [0] * size
        if not init_wavelength:
            wavelength = size
        else:
            wavelength = init_wavelength
        for i in range(level):
            amplitude = random.uniform(-1, 1) / (i + 1)**smoothing

            for e in range(size):
                data[e] += sin(e/wavelength * G_2PI) * amplitude

            wavelength *= 0.5

        plt.plot(data)
        plt.grid()
        plt.show()

    return data

create_ripples(1000, 8) # 1D
create_ripples([100, 100], 10, 1.8, use_numpy=True) # 2D


from gan import Descriminator,Generator
from util import train_step, test, visualize_stat
import tensorflow as tf
from classify import classify
from create_data import load_data

tf.keras.backend.clear_session()
train_ds, valid_ds, test_ds = load_data()

disc = Descriminator()
gen = Generator()

train_gen_losses = []
valid_gen_losses = []
train_disc_losses = []
valid_disc_losses = []
#valid_accuracies = []

with tf.device('/device:gpu:0'):
# training the model
    results,trained_gen,trained_disc = classify(gen,disc, 10, train_ds, valid_ds)

    # saving results for visualization
    train_gen_losses.append(results[0])
    valid_gen_losses.append(results[1])
    train_disc_losses.append(results[2])
    valid_disc_losses.append(results[3])
    #valid_accuracies.append(results[2])

# testing the trained model
# (this code snippet should only be inserted when one decided on all hyperparameters)
#_, test_accuracy = test(trained_model, test_ds, tf.keras.losses.BinaryCrossentropy(), False)
#print("Accuracy (test set):", test_accuracy)

# visualizing losses and accuracy
visualize_stat(train_gen_losses, valid_gen_losses, train_disc_losses,valid_disc_losses)


from itertools import product
import torch
import torch.nn.functional as functional

def spherical_to_cartesian(yaw, elevation):
    """Helper function to convert from a spherical coordinate system
    parameterized by a yaw and elevation to the xyz cartesian coordinate
    with a unit radius, where the z-axis points upwards.

    Arguments:

    yaw: torch.Tensor
        a tensor representing the top-down yaw in radians of the coordinate,
        starting from the positive x-axis and turning counter-clockwise.
    elevation: torch.Tensor
        a tensor representing the elevation in radians of the coordinate,
        about the x-axis, with positive corresponding to upwards tilt.

    Returns:

    point: torch.Tensor
        a tensor corresponding to a point specified by the given yaw and
        elevation, in spherical coordinates.

    """

    # zero elevation and zero yaw points along the positive x-axis
    return torch.stack([torch.cos(yaw) * torch.cos(elevation),
                        torch.sin(yaw) * torch.cos(elevation),
                        torch.sin(elevation)], dim=-1)

def project_camera_rays(image_height, image_width,
                        focal_length_y, focal_length_x,
                        dtype=torch.float32, device='cpu'):
    """Generate a ray for each pixel in an image with a particular map_height
    and map_width by setting up a pinhole camera and sending out a ray from
    the camera to the imaging plane at each pixel location

    Arguments:

    image_height: int
        an integer that described the map_height of the imaging plane in
        pixels and determines the number of rays sampled vertically
    image_width: int
        an integer that described the map_width of the imaging plane in
        pixels and determines the number of rays sampled horizontally
    focal_length_y: float
        the focal length of the pinhole camera, which corresponds to
        the distance to the imaging plane in units of y pixels
    focal_length_x: float
        the focal length of the pinhole camera, which corresponds to
        the distance to the imaging plane in units of x pixels

    Returns:

    rays: torch.Tensor
        a tensor that represents the directions of sampled rays in
        the coordinate system of the camera with shape: [height, width, 3]

    """

    # generate pixel locations for every ray in the imaging plane
    # where the returned shape is: [image_height, image_width]
    kwargs = dict(dtype=dtype, device=device)
    y, x = torch.meshgrid(torch.arange(image_height, **kwargs),
                          torch.arange(image_width, **kwargs), indexing='ij')

    # convert pixel coordinates to the camera coordinate system
    # y is negated to conform to OpenGL convention in computer graphics
    rays_y = (y - 0.5 * float(image_height - 1)) / focal_length_y
    rays_x = (x - 0.5 * float(image_width - 1)) / focal_length_x
    return torch.stack([rays_x, -rays_y, -torch.ones_like(rays_x)], dim=-1)

def transform_rays(rays, eye_vector, up_vector):
    """Given a batch of camera orientations, specified with a viewing
    direction and up vector, convert rays from the camera coordinate
    system to the world coordinate system using a rotation matrix.

    Arguments:

    rays: torch.Tensor
        a batch of rays that have been generated in the coordinate system
        of the camera with shape: [batch, map_height, map_width, 3]
    eye_vector: torch.Tensor
        a batch of viewing directions that are represented as three
        vectors in the world coordinate system with shape: [batch, 3]
    up_vector: torch.Tensor
        a batch of up directions in the imaging plane represented as
        three vectors in the world coordinate system with shape: [batch, 3]

    Returns:

    rays: torch.Tensor
        a batch of rays that have been converted to the coordinate system
        of the world with shape: [batch, map_height, map_width, 3]

    """

    # create a rotation matrix that transforms rays from the camera
    # coordinate system to the world coordinate system
    rotation = torch.stack([torch.cross(
        eye_vector, up_vector), up_vector, -eye_vector], dim=-1)

    # transform the rays using the rotation matrix such that rays project
    # out of the camera in world coordinates in the viewing direction
    return (rays.unsqueeze(-2) *
            rotation.unsqueeze(-3).unsqueeze(-3)).sum(dim=-1)

def bin_rays(bins0, bins1, bins2, origin, rays, depth,
             *features, min_ray_depth=0.0, max_ray_depth=10.0):
    """Given a set of rays and bins that specify the location and size of a
    grid of voxels, return the index of which voxel the end of each ray
    falls into, using a map_depth image to compute this point.

    Arguments:

    bins0: torch.FloatTensor
        a 1D tensor whose elements specify the locations of boundaries of a
        set of voxels along the 0th axis of the coordinate system.
    bins1: torch.FloatTensor
        a 1D tensor whose elements specify the locations of boundaries of a
        set of voxels along the 1st axis of the coordinate system.
    bins2: torch.FloatTensor
        a 1D tensor whose elements specify the locations of boundaries of a
        set of voxels along the 2nd axis of the coordinate system.

    origin: torch.FloatTensor
        the origin of the rays in world coordinates, represented as a batch
        of 3-vectors, shaped like [batch_size, 3].
    rays: torch.FloatTensor
        rays projecting outwards from the origin, ending at a point specified
        by the map_depth, shaped like: [batch_size, height, width, 3].
    map_depth: torch.FloatTensor
        the length of the corresponding ray in world coordinates before
        intersecting a surface, shaped like: [batch_size, height, width, 1].

    features: List[torch.Tensor]
        a list of features for every pixel in the image, such as class
        probabilities shaped like: [batch_size, height, width, num_features]

    min_ray_depth: float
        the minimum distance rays can be to the camera focal point, used to
        handle special cases, such as when the distance is zero.
    max_ray_depth: float
        the maximum distance rays can be to the camera focal point, used to
        handle special cases, such as when the distance is infinity.

    Returns:

    ind0: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 0, shaped like: [batch_size, num_points].
    ind1: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 1, shaped like: [batch_size, num_points].
    ind2: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 2, shaped like: [batch_size, num_points].

    ratio0: torch.FloatTensor
        fraction through each bin corresponding to the points in ind0,
        represented as a float tensor in the range [0, 1].
    ratio1: torch.FloatTensor
        fraction through each bin corresponding to the points in ind1,
        represented as a float tensor in the range [0, 1].
    ratio2: torch.FloatTensor
        fraction through each bin corresponding to the points in ind2,
        represented as a float tensor in the range [0, 1].

    binned_features: List[torch.Tensor]
        a list of features for every pixel in the image, such as class
        probabilities, shaped like: [batch_size, num_points, num_features].

    """

    # bin the point cloud according to which voxel points occupy in space
    # the xyz convention must be known by who is using the function
    rays = origin.unsqueeze(-2).unsqueeze(-2) + rays * depth
    ind0 = torch.bucketize(rays[..., 0].contiguous(), bins0, right=True) - 1
    ind1 = torch.bucketize(rays[..., 1].contiguous(), bins1, right=True) - 1
    ind2 = torch.bucketize(rays[..., 2].contiguous(), bins2, right=True) - 1

    # certain rays will be out of bounds of the map or will have a special
    # map_depth value used to signal invalid points, identify them
    criteria = [torch.logical_and(torch.ge(ind0, 0),
                                  torch.lt(ind0, bins0.size(dim=0) - 1)),
                torch.logical_and(torch.ge(ind1, 0),
                                  torch.lt(ind1, bins1.size(dim=0) - 1)),
                torch.logical_and(torch.ge(ind2, 0),
                                  torch.lt(ind2, bins2.size(dim=0) - 1))]

    # merge each of the criteria into a single mask tensor that will
    # have the shape [batch, image_height, image_width]
    criterion = torch.logical_and(torch.ge(depth.squeeze(-1), min_ray_depth),
                                  torch.le(depth.squeeze(-1), max_ray_depth))
    for next_criterion in criteria:
        criterion = torch.logical_and(criterion, next_criterion)

    # indices where the criterion is true and binned points are valid
    indices = torch.nonzero(criterion, as_tuple=True)

    # select the indices and ray coordinates
    ind0, ind1, ind2, rays = ind0[  # that are within the world bounds
        indices], ind1[indices], ind2[indices], rays[indices]

    # lookup left and right bounds for the voxel each point is binned to
    bounds0 = functional.embedding(torch.stack([
        ind0, ind0 + 1], dim=0), bins0.unsqueeze(-1)).squeeze(-1)
    bounds1 = functional.embedding(torch.stack([
        ind1, ind1 + 1], dim=0), bins1.unsqueeze(-1)).squeeze(-1)
    bounds2 = functional.embedding(torch.stack([
        ind2, ind2 + 1], dim=0), bins2.unsqueeze(-1)).squeeze(-1)

    # evaluate the fraction through the bin each point is
    # which is used when aggregating points in the projective map
    ratio0 = (rays[..., 0] - bounds0[0]) / (bounds0[1] - bounds0[0])
    ratio1 = (rays[..., 1] - bounds1[0]) / (bounds1[1] - bounds1[0])
    ratio2 = (rays[..., 2] - bounds2[0]) / (bounds2[1] - bounds2[0])

    # reverse the indices of the bins along the y axis according
    ind1 = bins1.size(dim=0) - 2 - ind1  # to the open gl convention

    # select a subset of the voxel indices and voxel feature predictions
    # in order to remove points that correspond to invalid voxels
    return (ind0, ind1, ind2, ratio0, 1.0 - ratio1, ratio2,
            *[features_i[indices] for features_i in features])

def update_feature_map(ind0, ind1, ind2, ratio0, ratio1, ratio2,
                       features, feature_map, interpolation_weight=1.0):
    """Scatter add feature vectors associated with a point cloud onto a
    voxel feature map by adding the features to the locations of each voxel
    using the voxel ids returned by the bin_rays function.

    Arguments:

    ind0: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 0, shaped like: [batch_size, num_points].
    ind1: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 1, shaped like: [batch_size, num_points].
    ind2: torch.LongTensor
        voxel ids associated with the point cloud according to their position
        along axis 2, shaped like: [batch_size, num_points].

    ratio0: torch.FloatTensor
        fraction through each bin corresponding to the points in ind0,
        represented as a float tensor in the range [0, 1].
    ratio1: torch.FloatTensor
        fraction through each bin corresponding to the points in ind1,
        represented as a float tensor in the range [0, 1].
    ratio2: torch.FloatTensor
        fraction through each bin corresponding to the points in ind2,
        represented as a float tensor in the range [0, 1].

    features: torch.Tensor
        tensor of features that will be added to the feature map using
        torch.scatter_add on the map: [batch_size, num_points, feature_dim].
    feature_map: torch.Tensor
        tensor of features organized as a three dimensional grid of
        voxels with shape: [batch_size, height, width, depth, num_features].
    interpolation_weight: float
        float representing the interpolation weight used when adding
        new features in the feature map weighted by interpolation_weight.

    """

    # get the size of the spatial dimensions of the map, used to
    # infer the batch size of the map if present
    size0, size1, size2, num_features = feature_map.shape[-4:]
    feature_map_flat = feature_map.view(
        -1, size0 * size1 * size2, num_features)

    # compute the indices of upper and lower voxels for each
    ind0_lower = torch.where(ratio0 < 0.5,  # point in the point cloud
                             (ind0 - 1).clamp(min=0), ind0)
    ind1_lower = torch.where(ratio1 < 0.5,
                             (ind1 - 1).clamp(min=0), ind1)
    ind2_lower = torch.where(ratio2 < 0.5,
                             (ind2 - 1).clamp(min=0), ind2)
    ind0_upper = torch.where(ratio0 < 0.5, ind0,
                             (ind0 + 1).clamp(max=size0 - 1))
    ind1_upper = torch.where(ratio1 < 0.5, ind1,
                             (ind1 + 1).clamp(max=size1 - 1))
    ind2_upper = torch.where(ratio2 < 0.5, ind2,
                             (ind2 + 1).clamp(max=size2 - 1))

    # concatenate all combinations of voxel indices along three axes
    indices = torch.cat([  # increasing num points by a factor of eight
        (ind0 * size1 + ind1) * size2 + ind2
        for ind0, ind1, ind2 in product((ind0_lower, ind0_upper),
                                        (ind1_lower, ind1_upper),
                                        (ind2_lower, ind2_upper))], dim=-1)

    # expand the indices tensor with an additional axis so that
    indices = indices.unsqueeze(  # indices has the same shape as features
        -1).expand(*(list(indices.shape) + [num_features]))

    # compute the weights of upper and lower voxels for each
    weight0_lower = torch.where(ratio0 < 0.5,  # point in the point cloud
                                0.5 - ratio0, 1.5 - ratio0)
    weight1_lower = torch.where(ratio1 < 0.5,
                                0.5 - ratio1, 1.5 - ratio1)
    weight2_lower = torch.where(ratio2 < 0.5,
                                0.5 - ratio2, 1.5 - ratio2)
    weight0_upper = torch.where(ratio0 < 0.5,
                                ratio0 + 0.5, ratio0 - 0.5)
    weight1_upper = torch.where(ratio1 < 0.5,
                                ratio1 + 0.5, ratio1 - 0.5)
    weight2_upper = torch.where(ratio2 < 0.5,
                                ratio2 + 0.5, ratio2 - 0.5)

    # concatenate all combinations of weights along each axis
    all_weights = 1e-9 + torch.cat([  # increasing num points by eight
        (w0 * w1 * w2).unsqueeze(-1)
        for w0, w1, w2 in product((weight0_lower, weight0_upper),
                                  (weight1_lower, weight1_upper),
                                  (weight2_lower, weight2_upper))], dim=-2)

    # ensure that all tensors have a batch dimension and if one is not
    if len(indices.shape) < 3:  # present, add a new one with only one unit
        indices = indices.unsqueeze(0)
    if len(features.shape) < 3:
        features = features.unsqueeze(0)
    if len(all_weights.shape) < 3:
        all_weights = all_weights.unsqueeze(0)

    # compute the total amount of probability mass used when interpolating
    # new features for every voxel, used for normalization
    weights_sum = torch.zeros_like(feature_map_flat[..., :1])
    weights_sum.scatter_(-2, indices[..., :1],
                         all_weights, reduce="add")

    # for each new point in the point cloud of features, compute
    # what the interpolated feature in the feature map would be in isolation
    interpolated_features = (((1.0 - interpolation_weight * all_weights) *
                             feature_map_flat.gather(-2, indices)) +
                             (interpolation_weight *
                              all_weights * features.tile(1, 8, 1)))

    # zero the features at all observed voxels in the feature map
    # and assign voxels to the wighted average of new interpolated features
    feature_map_flat.scatter_(-2, indices, 0)
    feature_map_flat.scatter_(-2, indices, interpolated_features *
                              all_weights / weights_sum
                              .gather(-2, indices[..., :1]), reduce="add")


from unittest.mock import patch

import pytest

from crawler.rabbit.messages.parsers.base_message import BaseMessage

@pytest.fixture
def logger():
    with patch("crawler.rabbit.messages.parsers.base_message.LOGGER") as logger:
        yield logger

@pytest.fixture
def subject():
    return BaseMessage()

@pytest.mark.parametrize(
    "errors, headline",
    [
        [[], "No errors were reported during processing."],
        [["Error 1"], "1 error was reported during processing."],
        [["Error 1", "Error 2"], "2 errors were reported during processing."],
        [["Error 1", "Error 2", "Error 3"], "3 errors were reported during processing."],
        [["Error 1", "Error 2", "Error 3", "Error 4"], "4 errors were reported during processing."],
        [["Error 1", "Error 2", "Error 3", "Error 4", "Error 5"], "5 errors were reported during processing."],
    ],
)
def test_textual_errors_summary_is_accurate_for_up_to_5_errors(subject, errors, headline):
    subject._textual_errors = errors
    assert subject.textual_errors_summary == [headline] + errors

def test_textual_errors_summary_is_accurate_for_6_errors(subject):
    subject._textual_errors = ["Error 1", "Error 2", "Error 3", "Error 4", "Error 5", "Error 6"]
    assert subject.textual_errors_summary == [
        "6 errors were reported during processing. Only the first 5 are shown.",
        "Error 1",
        "Error 2",
        "Error 3",
        "Error 4",
        "Error 5",
    ]

@pytest.mark.parametrize(
    "textual_errors, error_count",
    [[[], 0], [["Error 1"], 1], [["Error 1", "Error 2"], 2], [["Error 1", "Error 2", "Error 3"], 3]],
)
def test_log_error_count_logs_error_count_correctly(subject, logger, textual_errors, error_count):
    subject._textual_errors = textual_errors

    subject.log_error_count()

    logger.debug.assert_called_once()
    log_message = logger.debug.call_args.args[0]
    assert str(error_count) in log_message


import shutil
import typing

import fastapi
import fastapi.testclient
import sqlalchemy
import pytest

from zip2addr import (
    constants,
    db,
    main,
)
# from zip2addr.routers import zipcode as TT

CLIENT = fastapi.testclient.TestClient(main.APP)

@pytest.fixture(scope="function", name="my_db")
def db_session(my_datadir, tmp_path):
    """
    .. seealso::

       - https://fastapi.tiangolo.com/ja/advanced/testing-database/
       - zip2addr.db.get_session_ctx
    """
    db_path = tmp_path / constants.DATABASE_FILENAME
    shutil.copyfile(my_datadir / constants.DATABASE_FILENAME, db_path)

    dbs = db.get_session_class(db_path)()

    def get_session():
        """For tests.
        """
        try:
            yield dbs
        except sqlalchemy.exc.SQLAlchemyError as exc:
            assert exc is not None
            dbs.rollback()
        finally:
            dbs.rollback()
            dbs.close()

    main.APP.dependency_overrides[
        db.get_default_session
    ] = get_session

    yield dbs

def test_get_zipcodes(my_db):
    resp = CLIENT.get("/zipcodes/")
    assert resp.status_code == 200
    assert resp.json()

def dicts_have_same_key_value_pairs(
    lhs: dict, rhs: dict,
    keys: typing.Optional[tuple[str]] = None
) -> bool:
    """Test if dicts have same key and value pairs.

    .. note::

        It's not true always lhs == rhs because rhs may have key and value
        pairs lhs does not have.
    """
    if keys is None:
        keys = lhs.keys()

    for key in keys:
        if key not in lhs:
            return False

        if key not in rhs or rhs[key] != lhs[key]:
            return False

    return True

# .. seealso:: Actual zipcode data, e.g. *.csv, in tests/data/.
@pytest.mark.parametrize(
    ("zipcode", "status_code", "expected"),
    (("0000000", fastapi.status.HTTP_404_NOT_FOUND, None),
     ("9071801", fastapi.status.HTTP_200_OK,
      dict(
          zipcode="9071801",
          pref="沖縄県",
          city_ward="八重山郡　与那国町",
          house_numbers="与那国",
          roman_pref="OKINAWA KEN",
          roman_city_ward="YAEYAMA GUN YONAGUNI CHO",
          roman_house_numbers="YONAGUNI",
          kana_pref="ｵｷﾅﾜｹﾝ",
          kana_city_ward="ﾔｴﾔﾏｸﾞﾝﾖﾅｸﾞﾆﾁｮｳ",
          kana_house_numbers="ﾖﾅｸﾞﾆ",
      )),
     )
)
def test_get_zipcode(zipcode, status_code, expected, my_db):
    """
    .. todo::

       - Use response_model
       - Make the API returns all of address info along with zipcode
        - Maybe we should normalize the data in database files.
    """
    resp = CLIENT.get(f"/zipcodes/{zipcode}")
    assert resp.status_code == status_code
    if expected is not None:
        # assert expected == resp.json()
        assert dicts_have_same_key_value_pairs(
            expected, resp.json(),
            ("zipcode", "pref")
        )

@pytest.mark.parametrize(
    ("partial_zipcode", "min_n_results"),
    (("000", -1),
     ("907", 1),
     )
)
def test_get_zipcodes_by_partial_zipcode(
    partial_zipcode, min_n_results, my_db
):
    resp = CLIENT.get(f"/zipcodes/partial/{partial_zipcode}")
    assert resp.status_code == fastapi.status.HTTP_200_OK
    if min_n_results < 0:
        assert not resp.json()
    else:
        assert len(resp.json()) >= min_n_results


# ----------------------------------------
# Importing all necessary python libraries
import pandas as pd
import networkx as nx
import numpy as np
import matplotlib.pyplot as plt
import pickle
from sklearn.neighbors import KNeighborsClassifier

from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

from sklearn.naive_bayes import GaussianNB

from sklearn.tree import DecisionTreeClassifier
# ----------------------------------------

# ----------------------------------------
def getNHopNeighbors(node, hop, adjList): # It is simply a bfs till nhop, not on whole graph
    neighborsTillHop, n_neighbors = set(), {node}

    for i in range(hop):
        temp = set()

        for curNode in n_neighbors:
            if curNode in adjList:
                temp = temp.union(set(adjList[curNode]))

        neighborsTillHop = neighborsTillHop.union(temp)
        n_neighbors = temp

    return neighborsTillHop
# ----------------------------------------

# ----------------------------------------
def OneHotEncoding(Vertices_attributes):
    attributes = Vertices_attributes.replace(0, np.nan) # get_dummies ignores nan but does not ignore zero, we want to ignore 0 columns
    oneHot = pd.get_dummies(attributes, columns = attributes.columns)

    distinctLabelCount = dict()
    for col in Vertices_attributes.columns:
        # count prefix match
        count = 0
        for one_hot_col in oneHot.columns:
            if one_hot_col.startswith(col):
                count = count + 1
        distinctLabelCount[col] = count

    return oneHot, distinctLabelCount
# ----------------------------------------

# ----------------------------------------
def error_computation(numNodes, n_hop_neighbors, Vertices_attributes_oneHot, Vertices_attributes, adjList, distinctLabelCount):
    all_nodes_distribution = np.zeros((numNodes, len(Vertices_attributes_oneHot.columns)))

    for eachNode in range(numNodes):
        Immediate_friends_Nodes = getNHopNeighbors(eachNode, n_hop_neighbors, adjList) # gets a list of adjacent nodes till a hop; this method is written on allMethods.py
        Vertices_attributes_sum = Vertices_attributes_oneHot.iloc[list(Immediate_friends_Nodes)].sum()
        all_nodes_distribution[eachNode] = Vertices_attributes_sum.to_numpy()

    i = 0
    for col in Vertices_attributes.columns:
        count = distinctLabelCount[col]
        denominator = np.sum(all_nodes_distribution[ : , i : i+count], axis = 1)
        denominator = denominator[None].T
        all_nodes_distribution[ : , i : i+count] = all_nodes_distribution[ : , i : i+count] / denominator

        i = i + count

    all_nodes_distribution[np.isnan(all_nodes_distribution)] = 0 # replace NaN i.e. 0/0 witn 0

    return all_nodes_distribution
# ----------------------------------------

# ----------------------------------------
def accuracyMeasurement(Test_True_Labels, predicted_labels):
    matched = 0
    for i in range(len(Test_True_Labels)):
        if Test_True_Labels[i] == predicted_labels[i]:
            matched = matched + 1
    accuracy = (matched / len(Test_True_Labels)) * 100

    # confusion_matrix = dict()
    # for i in range(len(Test_True_Labels)):
    #     actual = Test_True_Labels[i]
    #     predicted = predicted_labels[i]
    #     if (actual, predicted) not in confusion_matrix:
    #         confusion_matrix[(actual, predicted)] = 0
    #     confusion_matrix[(actual, predicted)] = confusion_matrix[(actual, predicted)] + 1

    # total, matched = 0, 0
    # for each_key in confusion_matrix:
    #     total = total + confusion_matrix[each_key]
    #     if each_key[0] == each_key[1]: matched = matched + 1

    # accuracy = (matched / total) * 100
    return accuracy
# ----------------------------------------

# ----------------------------------------
def knnClassifier(Train, Test, Train_True_Labels):
    knn = KNeighborsClassifier(n_neighbors = 10) # Expected n_neighbors <= n_samples,  but n_samples = 4, n_neighbors = 10
    knn.fit(Train, Train_True_Labels)
    predicted_labels = knn.predict(Test)
    return predicted_labels
# ----------------------------------------

# ----------------------------------------
def svmClassifier(Train, Test, Train_True_Labels):
    clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
    clf.fit(Train, Train_True_Labels)
    predicted_labels = clf.predict(Test)
    return predicted_labels
# ----------------------------------------

# ----------------------------------------
def naiveBiasClassifier(Train, Test, Train_True_Labels):
    gnb = GaussianNB()
    predicted_labels = gnb.fit(Train, Train_True_Labels).predict(Test) # gnb.fit(X_train, y_train).predict(X_test)
    return predicted_labels
# ----------------------------------------

# ----------------------------------------
def decisionTreeClassifier(Train, Test, Train_True_Labels):
    dlf = DecisionTreeClassifier(random_state=48)
    predicted_labels = dlf.fit(Train, Train_True_Labels).predict(Test) # gnb.fit(X_train, y_train).predict(X_test)
    return predicted_labels
# ----------------------------------------


_base_ = '../../../benchmarks/classification/imagenet/vit-base-p16_linear-8xb2048-coslr-90e_in1k.py'  # noqa: E501

train_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(type='mmcls.ToPIL', to_rgb=True),
    dict(type='mmselfsup.MAERandomResizedCrop', size=224, interpolation=3),
    dict(type='mmcls.torchvision/RandomHorizontalFlip', p=0.5),
    dict(type='mmcls.ToNumpy', to_rgb=True),
    dict(type='PackClsInputs'),
]
train_dataloader = dict(
    batch_size=2048, dataset=dict(pipeline=train_pipeline), drop_last=True)


"""API Endpoints"""
import logging
from typing import Union

import tensorflow as tf
from fastapi import FastAPI
from fastapi import File
from fastapi import HTTPException
from fastapi import Query
from fastapi import status
from fastapi import UploadFile
from fastapi.middleware.cors import CORSMiddleware

from . import dependencies
from .. import config
from .. import schemas

#################### Initialization ####################

api = FastAPI(
    title="Dogs vs Cat API",
    description="Classify pictures between cats and dogs",
    version=config.Config.VERSION,
)

logger = tf.get_logger()
logger.setLevel(logging.ERROR)

#################### Middleware ####################

api.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["GET", "POST", "PUT", "DELETE"],
    allow_headers=["*"],
)

#################### Endpoints ####################

@api.post(
    "/classifier/cats-and-dogs",
    response_model=schemas.CatsAndDogsClassifierResponse,
    status_code=status.HTTP_200_OK,
    responses={
        422: {"description": "Could not process the provided picture | The provided picture has an invalid extension"},
    },
)
async def classify(
    picture: UploadFile = File(..., description="Upload the picture of either a cat or a dog")
) -> schemas.CatsAndDogsClassifierResponse:

    extension: str = picture.filename.split(".")[-1]
    if extension not in ["jpg", "jpeg", "png"]:
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="The provided picture has an invalid extension",
        )

    image: Union[bytes, str] = await picture.read()
    if not isinstance(image, bytes):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Could not process the provided picture",
        )

    return schemas.CatsAndDogsClassifierResponse(predicted_class=dependencies.image_classifier.predict(image=image))


class Solution:
    def uniqueOccurrences(self, arr: List[int]) -> bool:
        counts = []
        counted = []
        for i in arr:
            if i not in counted:
                counts.append(arr.count(i))
                counted.append(i)
        print(counts)
        for i in range(len(counts)):
            for j in range(len(counts)):
                if i != j :
                    if counts[i] == counts[j]:
                        return False
        return True


import math
from dataclasses import dataclass

@dataclass
class Abstract_Packet():
    version: int
    packetType: int

@dataclass
class Literal_Packet(Abstract_Packet):
    value: int

@dataclass
class Operator_Packet(Abstract_Packet):
    subs: list[Abstract_Packet]

def hex_to_binary(h):
    out = ""
    for digit in h:
        if digit.isnumeric():
            bi = decimal_to_binary(int(digit))
        else:
            bi = decimal_to_binary(ord(digit) - 55)
        for i in range(4-len(bi)):
            out += "0"
        out += bi
    return out

def decimal_to_binary(d):
    if d == 0:
        return "0"
    if d == 1:
        return "1"

    out = ""
    for power in range(math.floor(math.log(d, 2)), -1, -1):
        place = 2**power
        if place <= d:
            out += "1"
            d -= place
        else:
            out += "0"
    assert d == 0
    return out

def binary_to_decimal(b):
    out = 0
    for i, digit in enumerate(b[::-1]):
        if digit == "1":
            out += 2**i
    return out

def get_packets(infile):
    with open(infile) as f:
        data = f.read().strip()
    return hex_to_binary(data)

def parse_packets(packetData):
    if len(packetData) == 0 or packetData == "0"*len(packetData):
        return []
    packetType = binary_to_decimal(packetData[3:6])
    if packetType == 4:
        return parse_literal_packet(packetData)
    else:
        return parse_operator_packet(packetData)

def parse_literal_packet(packetData):
    version = binary_to_decimal(packetData[0:3])
    packetType = binary_to_decimal(packetData[3:6])

    binaryValue = ""
    valueIndex = 6
    valueEnd = False
    while not valueEnd:
        binaryValue += packetData[valueIndex+1:valueIndex+5]
        valueEnd = (packetData[valueIndex] == "0")
        valueIndex += 5
    value = binary_to_decimal(binaryValue)
    nextIndex = valueIndex
    packet = Literal_Packet(version, packetType, value)
    remainingPackets = packetData[nextIndex:]

    return [packet] + parse_packets(remainingPackets)

def parse_operator_packet(packetData):
    version = binary_to_decimal(packetData[0:3])
    packetType = binary_to_decimal(packetData[3:6])
    lengthTypeID = packetData[6]

    #count internal packets by size (15 bit number of bits)
    if lengthTypeID == "0":
        numBits = binary_to_decimal(packetData[7:22])
        internalPackets = parse_packets(packetData[22:22+numBits])

        packet = Operator_Packet(version, packetType, internalPackets)
        remainingPackets = packetData[22+numBits:]
        return [packet] + parse_packets(remainingPackets)

    #count internal packets by the number of packets
    else:
        numPackets = binary_to_decimal(packetData[7:18])
        allRemainingPackets = packetData[18:]
        parsedPackets = parse_packets(allRemainingPackets)
        internalPackets = parsedPackets[0:numPackets]
        remainingParsedPackets = parsedPackets[numPackets:]
        packet = Operator_Packet(version, packetType, internalPackets)
        return [packet] + remainingParsedPackets

def sum_versions(packets):
    versionSum = 0
    for packet in packets:
        versionSum += packet.version
        if type(packet) == Operator_Packet:
            versionSum += sum_versions(packet.subs)
    return versionSum

def get_packet_value(packet):

    #I don't have python 3.10 on my distro yet, so no match. Feelsbadman.
    if packet.packetType == 4:
        return packet.value

    elif packet.packetType == 0:
        return sum(map(get_packet_value, packet.subs))

    elif packet.packetType == 1:
        product = 1
        for sub in packet.subs:
            product *= get_packet_value(sub)
        return product

    elif packet.packetType == 2:
        return min(map(get_packet_value, packet.subs))

    elif packet.packetType == 3:
        return max(map(get_packet_value, packet.subs))

    elif packet.packetType == 5:
        subValues = list(map(get_packet_value, packet.subs))
        if subValues[0] > subValues[1]:
            return 1
        else:
            return 0

    elif packet.packetType == 6:
        subValues = list(map(get_packet_value, packet.subs))
        if subValues[0] < subValues[1]:
            return 1
        else:
            return 0

    elif packet.packetType == 7:
        subValues = list(map(get_packet_value, packet.subs))
        if subValues[0] == subValues[1]:
            return 1
        else:
            return 0

    else:
        raise(Exception(f"Invalid Packet Type: {packet.packetType}"))

def main():
    infile = "input.txt"
    packetData = get_packets(infile)
    packets = parse_packets(packetData)
    if len(packets) == 1:
        print(get_packet_value(packets[0]))
    else:
        print("Multiple packets.")

if __name__ == "__main__":
    main()


import unittest

import numpy as np

from pgmpy.factors.discrete import DiscreteFactor
from pgmpy.models import JunctionTree
from pgmpy.tests import help_functions as hf

class TestJunctionTreeCreation(unittest.TestCase):
    def setUp(self):
        self.graph = JunctionTree()

    def test_add_single_node(self):
        self.graph.add_node(("a", "b"))
        self.assertListEqual(list(self.graph.nodes()), [("a", "b")])

    def test_add_single_node_raises_error(self):
        self.assertRaises(TypeError, self.graph.add_node, "a")

    def test_add_multiple_nodes(self):
        self.graph.add_nodes_from([("a", "b"), ("b", "c")])
        self.assertListEqual(
            hf.recursive_sorted(self.graph.nodes()), [["a", "b"], ["b", "c"]]
        )

    def test_add_single_edge(self):
        self.graph.add_edge(("a", "b"), ("b", "c"))
        self.assertListEqual(
            hf.recursive_sorted(self.graph.nodes()), [["a", "b"], ["b", "c"]]
        )
        self.assertListEqual(
            sorted([node for edge in self.graph.edges() for node in edge]),
            [("a", "b"), ("b", "c")],
        )

    def test_add_single_edge_raises_error(self):
        self.assertRaises(ValueError, self.graph.add_edge, ("a", "b"), ("c", "d"))

    def test_add_cyclic_path_raises_error(self):
        self.graph.add_edge(("a", "b"), ("b", "c"))
        self.graph.add_edge(("b", "c"), ("c", "d"))
        self.assertRaises(ValueError, self.graph.add_edge, ("c", "d"), ("a", "b"))

    def tearDown(self):
        del self.graph

class TestJunctionTreeMethods(unittest.TestCase):
    def setUp(self):
        self.factor1 = DiscreteFactor(["a", "b"], [2, 2], np.random.rand(4))
        self.factor2 = DiscreteFactor(["b", "c"], [2, 2], np.random.rand(4))
        self.factor3 = DiscreteFactor(["d", "e"], [2, 2], np.random.rand(4))
        self.factor4 = DiscreteFactor(["e", "f"], [2, 2], np.random.rand(4))
        self.factor5 = DiscreteFactor(["a", "b", "e"], [2, 2, 2], np.random.rand(8))

        self.graph1 = JunctionTree()
        self.graph1.add_edge(("a", "b"), ("b", "c"))
        self.graph1.add_factors(self.factor1, self.factor2)

        self.graph2 = JunctionTree()
        self.graph2.add_nodes_from([("a", "b"), ("b", "c"), ("d", "e")])
        self.graph2.add_edge(("a", "b"), ("b", "c"))
        self.graph2.add_factors(self.factor1, self.factor2, self.factor3)

        self.graph3 = JunctionTree()
        self.graph3.add_edges_from([(("a", "b"), ("b", "c")), (("d", "e"), ("e", "f"))])
        self.graph3.add_factors(self.factor1, self.factor2, self.factor3, self.factor4)

        self.graph4 = JunctionTree()
        self.graph4.add_edges_from(
            [
                (("a", "b", "e"), ("b", "c")),
                (("a", "b", "e"), ("e", "f")),
                (("d", "e"), ("e", "f")),
            ]
        )
        self.graph4.add_factors(self.factor5, self.factor2, self.factor3, self.factor4)

    def test_check_model(self):
        self.assertRaises(ValueError, self.graph2.check_model)
        self.assertRaises(ValueError, self.graph3.check_model)
        self.assertTrue(self.graph1.check_model())
        self.assertTrue(self.graph4.check_model())

    def test_states(self):
        self.assertDictEqual(
            self.graph4.states,
            {
                "a": [0, 1],
                "b": [0, 1],
                "e": [0, 1],
                "c": [0, 1],
                "d": [0, 1],
                "f": [0, 1],
            },
        )

    def tearDown(self):
        del self.factor1
        del self.factor2
        del self.factor3
        del self.factor4
        del self.factor5

        del self.graph1
        del self.graph2
        del self.graph3
        del self.graph4

class TestJunctionTreeCopy(unittest.TestCase):
    def setUp(self):
        self.graph = JunctionTree()

    def test_copy_with_nodes(self):
        self.graph.add_nodes_from([("a", "b", "c"), ("a", "b"), ("a", "c")])
        self.graph.add_edges_from(
            [(("a", "b", "c"), ("a", "b")), (("a", "b", "c"), ("a", "c"))]
        )
        graph_copy = self.graph.copy()

        self.graph.remove_edge(("a", "b", "c"), ("a", "c"))
        self.assertFalse(self.graph.has_edge(("a", "b", "c"), ("a", "c")))
        self.assertTrue(graph_copy.has_edge(("a", "b", "c"), ("a", "c")))

        self.graph.remove_node(("a", "c"))
        self.assertFalse(self.graph.has_node(("a", "c")))
        self.assertTrue(graph_copy.has_node(("a", "c")))

        self.graph.add_node(("c", "d"))
        self.assertTrue(self.graph.has_node(("c", "d")))
        self.assertFalse(graph_copy.has_node(("c", "d")))

    def test_copy_with_factors(self):
        self.graph.add_edges_from([[("a", "b"), ("b", "c")]])
        phi1 = DiscreteFactor(["a", "b"], [2, 2], np.random.rand(4))
        phi2 = DiscreteFactor(["b", "c"], [2, 2], np.random.rand(4))
        self.graph.add_factors(phi1, phi2)
        graph_copy = self.graph.copy()

        self.assertIsInstance(graph_copy, JunctionTree)
        self.assertIsNot(self.graph, graph_copy)
        self.assertEqual(
            hf.recursive_sorted(self.graph.nodes()),
            hf.recursive_sorted(graph_copy.nodes()),
        )
        self.assertEqual(
            hf.recursive_sorted(self.graph.edges()),
            hf.recursive_sorted(graph_copy.edges()),
        )
        self.assertTrue(graph_copy.check_model())
        self.assertEqual(self.graph.get_factors(), graph_copy.get_factors())

        self.graph.remove_factors(phi1, phi2)
        self.assertTrue(
            phi1 not in self.graph.factors and phi2 not in self.graph.factors
        )
        self.assertTrue(phi1 in graph_copy.factors and phi2 in graph_copy.factors)

        self.graph.add_factors(phi1, phi2)
        self.graph.factors[0] = DiscreteFactor(["a", "b"], [2, 2], np.random.rand(4))
        self.assertNotEqual(self.graph.get_factors()[0], graph_copy.get_factors()[0])
        self.assertNotEqual(self.graph.factors, graph_copy.factors)

    def test_copy_with_factorchanges(self):
        self.graph.add_edges_from([[("a", "b"), ("b", "c")]])
        phi1 = DiscreteFactor(["a", "b"], [2, 2], np.random.rand(4))
        phi2 = DiscreteFactor(["b", "c"], [2, 2], np.random.rand(4))
        self.graph.add_factors(phi1, phi2)
        graph_copy = self.graph.copy()

        self.graph.factors[0].reduce([("a", 0)])
        self.assertNotEqual(
            self.graph.factors[0].scope(), graph_copy.factors[0].scope()
        )
        self.assertNotEqual(self.graph, graph_copy)
        self.graph.factors[1].marginalize(["b"])
        self.assertNotEqual(
            self.graph.factors[1].scope(), graph_copy.factors[1].scope()
        )
        self.assertNotEqual(self.graph, graph_copy)

    def tearDown(self):
        del self.graph


import boto3
from config import ConfigManager

aws_access_key_id = ConfigManager().get('default', 'DYNAMO_DB_AWS_ACCESS_KEY_ID')
aws_secret_access_key = ConfigManager().get('default', 'DYNAMO_DB_AWS_SECRET_SECRET_KEY')
region = ConfigManager().get('default', 'AWS_REGION')
# Create a DynamoDB client with your credentials
dynamodb = boto3.client('dynamodb',
                        region_name=region,
                        aws_access_key_id=aws_access_key_id,
                        aws_secret_access_key=aws_secret_access_key)

# test connection
try:
    response = dynamodb.list_tables()
    print(response)
except Exception as e:
    print(e)


# Should be considered a singleton representing a currently loaded project.

import json
import csv
import re
import os
import shutil
from collections import namedtuple, defaultdict
from main_utils import appPath
from dotmap import DotMap

import interop.jplag_preprocessor as jplag_preprocessor
import interop.jplag_runner as jplag_runner
import interop.moodle_downloader as moodle_downloader

_CurrentProjectPath = None  # initially no project file is loaded
_CurrentAssignment = None  # no detection performed

def _templatesDir():
    return appPath()  # may be revised

def settings():
    assert _CurrentProjectPath
    json_path = os.path.join(_CurrentProjectPath, "config.json")
    with open(json_path) as f:
        return DotMap(json.load(f))

def saveSettings(config):
    json_path = os.path.join(_CurrentProjectPath, "config.json")
    with open(json_path, "w") as f:
        json.dump(config, f, indent=4)

def newProject(project_path):  # initialize project at given path
    global _CurrentProjectPath
    _CurrentProjectPath = project_path
    shutil.copyfile(
        os.path.join(_templatesDir(), "config_template.json"),
        os.path.join(_CurrentProjectPath, "config.json"),
    )

def openProject(project_path):  # open project at given path
    global _CurrentProjectPath
    _CurrentProjectPath = project_path

def syncWithDataSource(progress_object):
    assert _CurrentProjectPath
    os.chdir(_CurrentProjectPath)
    moodle_downloader.download(settings(), progress_object)

def assignments():
    assert _CurrentProjectPath
    os.chdir(os.path.join(_CurrentProjectPath, settings().moodle_submissions_dir))
    r = [d for d in os.listdir() if os.path.isdir(d)]
    os.chdir(_CurrentProjectPath)
    return r

def _is_arc_student(student):
    return student.startswith("arc[")

def _filter_arc_records(log):
    def student_name(student_name):
        m = re.match("(arc\[.+\]).+", student_name)
        return m.group(1) if m else student_name

    # log is in a format
    # "<user1>-<user2>: <sim_ratio>\n<user1>-<user3>: <sim_ratio>\n..."
    # archived users are "arc[dirname]user"

    Match = namedtuple("Match", ["student1", "student2", "score"])

    log_matches = []  # first, remove all archive-archive matches
    for log_line in log.strip().split("\n"):
        m = re.match("(.+)-(.+): (.+)", log_line)
        log_matches.append(Match(m.group(1), m.group(2), float(m.group(3))))

    # next, combine all students of the same archive
    # use max score over the archive
    log_filtered = defaultdict(float)
    for m in log_matches:
        if not (_is_arc_student(m.student1) and _is_arc_student(m.student2)):
            key = (student_name(m.student1), student_name(m.student2))

            if key not in log_filtered:  # name pair not found
                key = tuple(reversed(key))  # swap names

            log_filtered[key] = max(log_filtered[key], m.score)

    # TODO: use dictionary as an input format for the matrix
    return "\n".join([f"{n1}-{n2}: {v}" for (n1, n2), v in log_filtered.items()])

def detect(asgn):
    global _CurrentAssignment
    assert _CurrentProjectPath

    cfg = settings()
    os.chdir(_CurrentProjectPath)
    _CurrentAssignment = asgn

    # do not run if we already have results
    if not os.path.exists(f"jpl_out_{asgn}.log"):
        jplag_preprocessor.preprocess_dirs(
            cfg.moodle_submissions_dir,
            cfg.archive_dirs,
            cfg.assignment_regexes,
            asgn,
        )
        jplag_runner.run(
            cfg.java_path,
            cfg.template_dir,
            cfg.language,
            cfg.min_sim,
            cfg.jplag_args,
            asgn,
        )

    with open(f"jpl_out_{asgn}.log") as f:
        return _filter_arc_records(f.read())

def _find_archive_matches(student1, student2, asgn_path):
    # if one of the students is arc, we need to rename
    # it according to the best match from the log
    # there is some intersection with code above

    # here we simply have someone with archive prefix
    maxsim = 0
    r1 = r2 = ""
    with open(f"{asgn_path}.log") as f:
        for log_line in f.read().strip().split("\n"):
            m = re.match("(.+)-(.+): (.+)", log_line)
            name1, name2, sim = m.group(1), m.group(2), float(m.group(3))
            opt1 = name1.startswith(student1) and name2.startswith(student2)
            opt2 = name2.startswith(student1) and name1.startswith(student2)
            if sim > maxsim and (opt1 or opt2):
                maxsim, r1, r2 = sim, name1, name2

    return r1, r2

def htmlReportPath(student1, student2):
    assert _CurrentProjectPath
    assert _CurrentAssignment

    asgn_path = os.path.join(_CurrentProjectPath, f"jpl_out_{_CurrentAssignment}")
    csv_path = os.path.join(asgn_path, "pair_report.csv")

    if _is_arc_student(student1) or _is_arc_student(student2):
        student1, student2 = _find_archive_matches(student1, student2, asgn_path)

    # student1; student2; html-report
    with open(csv_path) as f:
        reader = csv.reader(f, delimiter=";")
        for row in reader:
            if (student1, student2) in [(row[0], row[1]), (row[1], row[0])]:
                return os.path.join(asgn_path, row[2])

    assert False, "match not found"

#### MP ####
# NOTE(mm): obsolete function, but referenced in one place
def viewMatchReport(studentID_1, studentID_2):
    # TODO: should be in main window
    pass

    # # TODO: always shows 'match0.html' -- for testing
    # html_path = os.path.join(
    #     _CurrentProjectPath, f"jpl_out_{_CurrentAssignment}", "match0.html"
    # )
    # isOk = MatchViewerDialog.show(_getMainWin(), studentID_1, studentID_2, html_path)
    # if isOk:
    #     print("Yay!")
    # CurrentAssignment
    # do whatever you want

def dataSource():
    # Здесь, по моему замыслу, настраивается доступ к Мудлу или его альтернативам.
    # Я могу сделать удобную форму для этого.
    # Идея в том, что если у тебя уже есть готовый проект, но что-то поменялось,
    # например, пароль к Мудлу или еще что-то,
    # ты можешь зайти сюда и поменять настройки.
    print("data source")

def importExportSettings():
    # Эта функция на случай, если ты уже все под себя настроил в одном проекте,
    # и не хочешь заниматься тем же самым в новом.
    # Думаю, можно просто дать возможность выбрать файл настроек из другого проекта,
    # и скопировать их в текущий. Понятно, что это не самая жизненно необходимая функция,
    # но очень может быть полезной, а сделать несложно… наверное.
    print("import and export settings")

def exportTemplate():
    # Да, здесь я признаю, увлеклась.
    # Даже точно не помню какой изначально был план :)
    # Наверное, это будет зависеть от того, что такое будет наш проект,
    # и что будет создаваться вначале. Но, вообще,
    # было бы неплохо иметь возможность загрузить некий каркас,
    # возможно с примерами, чтобы пользователь мог попробовать все, до того,
    # как переименует все файлы нужным образом, и еще так по-всякому по-разному заморозится.
    # Но, это, конечно та функция, которую ты ужалишь первой, и на том и будет, наш компромисс :D
    # Как-то так...
    print("export template")


import logging
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
import multiprocessing

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

source = './dataset/nlpcc/articles.pretrain.txt'

sentences = LineSentence(source)

# output model
output1 = './word2vec/word2vec.model'
# output vector
output2 = './word2vec/word2vec.vector'

def train():
    # train word2vec
    print('Start training...')
    model = Word2Vec(sentences=sentences, min_count=1, size=200, workers=multiprocessing.cpu_count())

    # save word2vec
    print('Save model to', output1)
    model.save(output1)
    print('Save model to', output2)
    model.wv.save_word2vec_format(output2)


import pygame

class cButton():
    """
    A class to represent Control Button object

    Attributes
    ------
    mouse : list
        position of user's mouse
    click : bool
        is mouse clicked
    smallText : python's font
        font for button's content
    textSurf : pygame's font
        attribute for pygame creating UI Text
    textRect : tuple
        width and height of text
    black : tuple
        color used for text
    gameDisplay : python's UI controller
        python's UI controller

    Methods
    ------
    Create(msg, x, y, w, h, ic, ac, action):
        Create a button with action when clicked
    """

    def __init__(self, gameDisplay):
        self.mouse = None
        self.click = None
        self.smallText = None
        self.textSurf = None
        self.textRect = None
        self.black = (0, 0, 0)
        self.gameDisplay = gameDisplay

    def Create(self, msg, x, y, w, h, ic, ac,  action=None):
        """
        Create new button for UI
        :param msg: String, text label
        :param x: double, position X
        :param y: double, position Y
        :param w: double, button width
        :param h: double, button height
        :param ic: tuple, color when not press
        :param ac: tuple, color when pressed
        :param action: function, action when button clicked
        :return: do action when button clicked
        """

        self.mouse = pygame.mouse.get_pos()
        self.click = pygame.mouse.get_pressed()
        print(self.click)
        if x+w > self.mouse[0] > x and y+h > self.mouse[1] > y:
            pygame.draw.rect(self.gameDisplay, ac, (x, y, w, h))

            if self.click[0] == 1 and action != None:
                action()
        else:
            pygame.draw.rect(self.gameDisplay, ic, (x, y, w, h))

        self.smallText = pygame.font.SysFont("comicsansms", 20)
        self.textSurf, self.textRect = self.instantiate_text(
            msg, self.smallText)
        self.textRect.center = ((x+(w/2)), (y+(h/2)))
        self.gameDisplay.blit(self.textSurf, self.textRect)

    def instantiate_text(self, text, font):
        """
        Display text
        :param text: String, the text to display
        :param font: pygame's system font , font of text
        :return: materials for python display text
        """
        textsurface = font.render(text, True, self.black)
        return textsurface, textsurface.get_rect()


from datetime import datetime
from os import getenv
from uuid import uuid4

from databases import Database
from sqlalchemy import Column, DateTime, String
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import (Delete, Insert, Select, Update, delete, insert,
                            select, update)
from sqlalchemy.sql.functions import func

DATABASE_URL = getenv("DATABASE_URL", "sqlite+aiosqlite:///database.db")
engine = create_async_engine(DATABASE_URL)
Base = declarative_base()

class Crud:
    id = Column(String(36), primary_key=True, index=True)
    created_at = Column(DateTime, nullable=False)
    updated_at = Column(DateTime, nullable=False)

    @classmethod
    async def execute(cls, stmt: str):
        async with Database(DATABASE_URL) as db:
            return await db.execute(stmt)

    @classmethod
    async def fetch_one(cls, stmt: str):
        async with Database(DATABASE_URL) as db:
            return await db.fetch_one(stmt)

    @classmethod
    async def fetch_all(cls, stmt: str):
        async with Database(DATABASE_URL) as db:
            return await db.fetch_all(stmt)

    @classmethod
    async def fetch_val(cls, stmt: str):
        async with Database(DATABASE_URL) as db:
            return await db.fetch_val(stmt)

    @classmethod
    def select(cls) -> Select:
        return select(cls)

    @classmethod
    def insert(cls) -> Insert:
        return insert(cls)

    @classmethod
    def update(cls) -> Update:
        return update(cls)

    @classmethod
    def delete(cls) -> Delete:
        return delete(cls)

    @classmethod
    def count(cls) -> Select:
        return select(func.count(cls.id))

    @classmethod
    async def create(cls, attrs: dict) -> str:
        attrs["id"] = str(uuid4())
        attrs["created_at"] = datetime.utcnow()
        attrs["updated_at"] = datetime.utcnow()
        await cls.execute(cls.insert().values(**attrs))
        return attrs["id"]

    @classmethod
    async def find_by_id(cls, id: str):
        return await cls.fetch_one(cls.select().where(cls.id == id))

    @classmethod
    async def find_all_by_attr(cls, attr, value, limit: int, offset: int):
        return await cls.fetch_all(cls.select().where(attr == value).limit(limit).offset(offset))

    @classmethod
    async def find_all_by_attr_no_limit(cls, attr, value):
        return await cls.fetch_all(cls.select().where(attr == value))

    @classmethod
    async def find_all(cls, limit: int, offset: int):
        return await cls.fetch_all(cls.select().limit(limit).offset(offset))

    @classmethod
    async def find_all_no_limit(cls):
        return await cls.fetch_all(cls.select())

    @classmethod
    async def count_by_attr(cls, attr, value):
        return await cls.fetch_val(cls.count().where(attr == value))

    @classmethod
    async def count_all(cls):
        return await cls.fetch_val(cls.count())

    @classmethod
    async def exist_by_id(cls, id: str):
        return await cls.find_by_id(id) is not None

    @classmethod
    async def update_by_id(cls, id: str, attrs: dict):
        attrs["updated_at"] = datetime.utcnow()
        return await cls.execute(cls.update().where(cls.id == id).values(**attrs))

    @classmethod
    async def delete_by_id(cls, id: str):
        return await cls.execute(cls.delete().where(cls.id == id))

    @classmethod
    async def delete_all_by_attr(cls, attr, value):
        return await cls.execute(cls.delete().where(attr == value))

class AuthorRelatedCrud(Crud):
    @classmethod
    async def find_author_id(cls, obj) -> str:
        raise NotImplementedError

class CourseRelatedCrud(Crud):
    @classmethod
    async def find_course_id(cls, obj) -> str:
        raise NotImplementedError()


from medplexity.benchmarks.healthsearchqa.healthsearchqa_dataset_builder import (
    HealthSearchQADatasetBuilder,
)
from medplexity.benchmarks.medicationqa.medicationqa_dataset_builder import (
    MedicationQADatasetBuilder,
)
from medplexity.benchmarks.medmcqa.medmcqa_dataset_builder import MedMCQADatasetBuilder
from medplexity.benchmarks.medqa.medqa_dataset_builder import MedQADatasetBuilder
from medplexity.benchmarks.mmlu.mmlu_dataset_builder import MMLUDatasetBuilder
from medplexity.benchmarks.mts_dialog import MTSDialogDatasetBuilder
from medplexity.benchmarks.pubmedqa.pubmedqa_dataset_builder import (
    PubmedQADatasetBuilder,
)
from medplexity.benchmarks.vqarad import VQARadDatasetBuilder

class DatasetFactory:
    DATASET_NAME_TO_BUILDER = {
        "healthsearchqa": HealthSearchQADatasetBuilder(),
        "medicationqa": MedicationQADatasetBuilder(),
        "medmcqa": MedMCQADatasetBuilder(),
        "medqa": MedQADatasetBuilder(),
        "mmlu": MMLUDatasetBuilder(),
        "pubmedqa": PubmedQADatasetBuilder(),
        "mts-dialog": MTSDialogDatasetBuilder(),
        "vqarad": VQARadDatasetBuilder(),
    }

    def build(self, name: str, split_type: str, config: dict | None = None):
        if name not in self.DATASET_NAME_TO_BUILDER:
            raise ValueError(f"Dataset {name} not supported.")

        return self.DATASET_NAME_TO_BUILDER[name].build_dataset(split_type, config)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  7 14:41:30 2019

@author: nk7g14
"""
import pandas as pd
import fluxquery.auxil as aux
import os
import requests

import logging
import re
from astropy.io import fits
import numpy as np
from pathlib import Path
import glob

logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s -- %(message)s')
class RXTE:
    def __init__(self):
        super(RXTE, self).__init__()
        self.RXTE_OBS_LIST = aux.GetObservationList(self.SOURCE_NAME, 'xtemaster')

    def _RXTE_DownloadObservation(self, obsID):
        print('Downloading RXTE observation:', obsID)
        filepath = '/sources/{}/rxte/{}.tar'.format(self.SOURCE_NAME, obsID)
        #RXTE observation IDS have a structure like XXXXX-XX-XX-XX
        #and are saved in folders on the archive with the first bit XXXXX with
        #a P in front of it.
        first_bit = obsID.split('-')[0]
        filepath_folder = 'P' + first_bit #Extracted OBSid folder

        tar_exists = Path(filepath).is_file()
        folder_exists = Path(filepath_folder).is_file()

        if tar_exists or folder_exists:
            print('Folder or tar file already exists, not downloading.')
        else:
            url = 'http://heasarc.gsfc.nasa.gov/cgi-bin/W3Browse/xteTar.pl?obsid={}&prnb={}'.format(obsID, first_bit)
            myfile = requests.get(url, allow_redirects=True)
            cwd = os.getcwd()
            open(cwd + filepath, 'wb').write(myfile.content)

    def RXTE_DownloadAllObservations(self):
        for obsID in self.RXTE_OBS_LIST['OBSID']:
            self._RXTE_DownloadObservation(obsID)

    def RXTE_GetCountsVpXPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['VpX1LCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['VpX1RCntPcu0'], dtype=float)
        count3 = np.array(xfl_file[1].data['VpX1LCntPcu1'], dtype=float)
        count4 = np.array(xfl_file[1].data['VpX1RCntPcu1'], dtype=float)
        count5 = np.array(xfl_file[1].data['VpX1LCntPcu2'], dtype=float)
        count6 = np.array(xfl_file[1].data['VpX1RCntPcu2'], dtype=float)
        count7 = np.array(xfl_file[1].data['VpX1LCntPcu3'], dtype=float)
        count8 = np.array(xfl_file[1].data['VpX1RCntPcu3'], dtype=float)
        count9 = np.array(xfl_file[1].data['VpX1LCntPcu4'], dtype=float)
        count10 = np.array(xfl_file[1].data['VpX1RCntPcu4'], dtype=float)

        df['Time'] = time
        df['VpX1LCntPcu0'] = count1
        df['VpX1RCntPcu0'] = count2
        df['VpX1LCntPcu1'] = count3
        df['VpX1RCntPcu1'] = count4
        df['VpX1LCntPcu2'] = count5
        df['VpX1RCntPcu2'] = count6
        df['VpX1LCntPcu3'] = count7
        df['VpX1RCntPcu3'] = count8
        df['VpX1LCntPcu4'] = count9
        df['VpX1RCntPcu4'] = count10
        return df

    def RXTE_GetCountsQ6VPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu1'], dtype=float)
        count3 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu2'], dtype=float)
        count4 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu3'], dtype=float)
        count5 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu4'], dtype=float)

        df['Time'] = time
        df['Q6VxVpXeCntPcu0'] = count1
        df['Q6VxVpXeCntPcu1'] = count2
        df['Q6VxVpXeCntPcu2'] = count3
        df['Q6VxVpXeCntPcu3'] = count4
        df['Q6VxVpXeCntPcu4'] = count5
        return df

    def RXTE_GetCountsXPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['X1LX2LCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['X1RX2RCntPcu0'], dtype=float)
        count3 = np.array(xfl_file[1].data['X2LX2RCntPcu0'], dtype=float)
        count4 = np.array(xfl_file[1].data['X2LX3LCntPcu0'], dtype=float)
        count5 = np.array(xfl_file[1].data['X2RX3RCntPcu0'], dtype=float)
        count6 = np.array(xfl_file[1].data['X3LX3RCntPcu0'], dtype=float)

        df['Time'] = time
        df['X1LX2LCntPcu0'] = count1
        df['X1RX2RCntPcu0'] = count2
        df['X2LX2RCntPcu0'] = count3
        df['X2LX3LCntPcu0'] = count4
        df['X2RX3RCntPcu0'] = count5
        df['X3LX3RCntPcu0'] = count6
        return df

    def RXTE_GetcountsGood(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'])
        count_good_1 = np.array(xfl_file[1].data['evXEgood_PCU0'], dtype=float)
        count_good_2 = np.array(xfl_file[1].data['evXEgood_PCU1'], dtype=float)
        count_good_3 = np.array(xfl_file[1].data['evXEgood_PCU2'], dtype=float)
        count_good_4 = np.array(xfl_file[1].data['evXEgood_PCU3'], dtype=float)
        count_good_5 = np.array(xfl_file[1].data['evXEgood_PCU4'], dtype=float)

        df['Time'] = time
        df['evXEgood_PCU0'] = count_good_1
        df['evXEgood_PCU1'] = count_good_2
        df['evXEgood_PCU2'] = count_good_3
        df['evXEgood_PCU3'] = count_good_4
        df['evXEgood_PCU4'] = count_good_5
        return df

    def RXTE_GetCounts(self, xfl_file):
        #Tempoarily removed good counts as for some observations don't have em
        countsVpX = self.RXTE_GetCountsVpXPcu(xfl_file)
        counts6VP = self.RXTE_GetCountsQ6VPcu(xfl_file)
        countsXP = self.RXTE_GetCountsXPcu(xfl_file)
        #counts_good = self.RXTE_GetcountsGood(xfl_file)

        #mapping = [countsVpX, counts6VP, countsXP, counts_good]
        mapping = [countsVpX, counts6VP, countsXP]
        df = pd.concat(mapping, axis=1)
        df = df.loc[:, ~df.columns.duplicated()] #Drop duplicate time columns
        return df

    def RXTE_GetAllCounts(self):
        '''
        Runs through all stdprod folders and obtains the flux for each obsID
        returns a dictionary containing all the flux dataframes
        '''
        walk = os.walk('sources/{}/rxte'.format(self.SOURCE_NAME))

        obsIDregex = re.compile(r'\d{5}-\d{2}-\d{2}-\d{2}')
        df_dict = {}

        gz_files = glob.glob('sources/{}/rxte/*/*/stdprod/*.xfl.gz'.format(self.SOURCE_NAME))
        xfl_files = glob.glob('sources/{}/rxte/*/*/stdprod/*.xfl'.format(self.SOURCE_NAME))

        len(gz_files)
        len(xfl_files)

        for walk_iter in walk:
            if 'stdprod' in walk_iter[0]:
                result = obsIDregex.search(walk_iter[0])
                obsID = result.group()
                logging.debug('Getting counts rxte %s', obsID)
                aux.UnzipAllgzFiles(walk_iter[0])
                xfl_search = glob.glob(walk_iter[0] + '/*.xfl')[0]
                xfl_file = fits.open(xfl_search)
                count_df = self.RXTE_GetCounts(xfl_file)
                df_dict[obsID] = count_df
        return df_dict

    def RXTE_MergeDataframeDictionary(self, df_dict):
        df = pd.concat(df_dict.values(), ignore_index=True)
        df = df.sort_values(by=['Time'])
        df = df.reset_index(drop=True)
        return df

    def RXTE_GetLightcurve(self):
        df_dict = self.RXTE_GetAllCounts()
        df = self.RXTE_MergeDataframeDictionary(df_dict)
        self.LIGHTCURVE_RXTE = df
        return df

    def RXTE_Complete(self):
        self.RXTE_DownloadAllObservations()
        aux.UnzipalltarFiles('sources/{}/rxte'.format(self.SOURCE_NAME))
        aux.RemoveAlltarFiles('sources/{}/rxte'.format(self.SOURCE_NAME))
        lc = self.RXTE_GetLightcurve()
        lc.plot(x='Time')
        return lc


import os
import re
import torch
import shutil

from enum import Enum
from collections import Counter
from safetensors.torch import load_file
from diffusers import StableDiffusionPipeline, AutoencoderKL

from .convert_vae_pt_to_diffusers import vae_pt_to_vae_diffuser
from .diffusion_error import ModelFormatError, ModelIdentificationError, ModelLoadError

class LoadModelType(Enum):
    SD_model_Diffusers      = 'Stable Diffusion Model (Diffusers Model)'
    SD_model_Safetensors    = 'Stable Diffusion Model (Safetensors Model)'
    SD_model_Ckpt           = 'Stable Diffusion Model (CKPT Model)'
    VAE_model_Diffusers     = 'Stable Diffusion Vae (Diffusers Model)'
    VAE_model_Pt            = 'Stable Diffusion Vae (PT Model)'
    LORA_model_Safetensors  = 'Stable Diffusion Lora (Safetensors Model)'
    HuggingFace             = 'Huggingface Repository Path'

class ModelType(Enum):
    SD_model = 'Stable Diffusion Model'
    VAE_model = 'Stable Diffusion Vae'
    LORA_model = 'Stable Diffusion Lora'

class BaseLoader():
    def __init__(self, dtype: torch.dtype = torch.float16, cache: str = './cache') -> None:
        self.dtype = dtype
        self.cache = cache
        self.basename = None
        self.basetype = None
        os.makedirs(cache, exist_ok=True)

    def determine_model_from_file(self, model_path: str) -> LoadModelType:
        def __logger(lmt: LoadModelType) -> LoadModelType:
            print(f'The model loaded was \"{lmt.value}\".')
            return lmt
        print('Determining model.')
        if os.path.isfile(model_path):
            _, ext = os.path.splitext(model_path)
            if ext == '.safetensors':
                dict = load_file(model_path)
                if any('diffusion_model' in key for key in dict.keys()):
                    return __logger(LoadModelType.SD_model_Safetensors)
                elif any('lora_unet' in key for key in dict.keys()):
                    return __logger(LoadModelType.LORA_model_Safetensors)
            elif ext == '.ckpt':
                return __logger(LoadModelType.SD_model_Ckpt)
            elif ext == '.pt':
                return __logger(LoadModelType.VAE_model_Pt)
        elif os.path.isdir(model_path):
            vlist = ['config.json', 'diffusion_pytorch_model.safetensors']
            flist = os.listdir(model_path)
            if Counter(flist) == Counter(vlist):
                return __logger(LoadModelType.VAE_model_Diffusers)
            else:
                return __logger(LoadModelType.SD_model_Diffusers)
        elif re.fullmatch(r'^([^/]*?)/([^/]*?)$', model_path):
            return __logger(LoadModelType.HuggingFace)
        raise ValueError('The specified argument is malformed.')

    def remove_model(self, model_path:str) -> (str, ModelType):
        model_type = self.determine_model_from_file(model_path)

        if model_type != LoadModelType.HuggingFace:
            cache_path = os.path.abspath(self.cache)
            target_path = os.path.abspath(model_path)
            if not os.path.commonprefix([target_path, cache_path]) == cache_path:
                raise ValueError('The specified path does not exist in the cache directory.')

        if model_type in [LoadModelType.SD_model_Diffusers, LoadModelType.VAE_model_Diffusers]:
            shutil.rmtree(model_path)
        elif model_type in [LoadModelType.SD_model_Safetensors,
                            LoadModelType.LORA_model_Safetensors,
                            LoadModelType.SD_model_Ckpt,
                            LoadModelType.VAE_model_Pt]:
            os.remove(model_path)
        elif model_type == LoadModelType.HuggingFace:
            names = model_path.split('/')
            shutil.rmtree(os.path.join(self.cache, f'models--{names[0]}--{names[1]}'))
        else:
            raise ModelIdentificationError('Could not identify type of model to delete.')
        return model_path

    def set_model(self, model_path: str) -> (str, ModelType):
        pass

    def get_path(self):
        return self.basename

    def get_type(self):
        return self.basetype

class DiffusionModelLoader(BaseLoader):
    def __model_cacher(self, model_path: str) -> str:
        path = os.path.join(self.cache, os.path.basename(model_path) + '_from_file')
        pipe = StableDiffusionPipeline.from_single_file(
            pretrained_model_link_or_path=model_path,
            torch_dtype=self.dtype,
            cache_dir=self.cache
        )
        pipe.save_pretrained(
            save_directory=path,
            safe_serialization=True
        )
        return path

    def __model_check_and_buffer(self, model_path: str) -> bool:
        try:
            pipe = StableDiffusionPipeline.from_pretrained(
                pretrained_model_name_or_path=model_path,
                torch_dtype=self.dtype,
                cache_dir=self.cache)
        except:
            return False
        return True

    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if not modelType in [LoadModelType.SD_model_Ckpt,
                             LoadModelType.SD_model_Safetensors,
                             LoadModelType.SD_model_Diffusers,
                             LoadModelType.HuggingFace]:
            raise ModelFormatError('The specified model is not a Diffusion model.')

        if modelType in [LoadModelType.SD_model_Safetensors,
                         LoadModelType.SD_model_Ckpt]:
            print('Creating cache from file.')
            self.basename = self.__model_cacher(model_path)
        elif modelType == LoadModelType.SD_model_Diffusers:
            print('Creating cache from Diffusers model.')
            if not self.__model_check_and_buffer(model_path):
                raise ModelLoadError('The specified path could not be loaded as a Diffusion model.')
            path = os.path.join(self.cache, os.path.basename(model_path))
            shutil.copytree(model_path, path)
            self.basename = path
        elif modelType == LoadModelType.HuggingFace:
            print('Creating cache from HuggingFace.')
            if not self.__model_check_and_buffer(model_path):
                raise ModelLoadError('Could not successfully retrieve model from HuggingFace.')
            self.basename = model_path

        self.basetype = ModelType.SD_model
        return self.basename, self.basetype

class VaeModelLoader(BaseLoader):
    def __model_checker(self, model_path: str) -> bool:
        try:
            pipe = AutoencoderKL.from_pretrained(
                pretrained_model_name_or_path=model_path,
                torch_dtype=self.dtype,
                cache_dir=self.cache)
        except:
            return False
        return True

    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if not modelType in [LoadModelType.VAE_model_Diffusers,
                             LoadModelType.VAE_model_Pt]:
            raise ModelFormatError('The specified model is not a Vae model.')

        if modelType == LoadModelType.VAE_model_Diffusers:
            print('Creating cache from Diffusers model.')
            if not self.__model_checker(model_path):
                raise ModelLoadError('The specified path could not be loaded as a Vae model.')
            path = os.path.join(self.cache, os.path.basename(model_path))
            shutil.copytree(model_path, path)
            self.basename = path
        elif modelType == LoadModelType.VAE_model_Pt:
            print('Creating cache from pt model.')
            path = os.path.join(self.cache, os.path.basename(model_path) + '_from_file')
            vae_pt_to_vae_diffuser(model_path, path)
            self.basename = path
        self.basetype = ModelType.VAE_model
        return self.basename, self.basetype

class LoraModelLoader(BaseLoader):
    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if modelType != LoadModelType.LORA_model_Safetensors:
            raise ModelFormatError('The specified model is not a Lora model.')
        print('Copying file to cache.')
        path = os.path.join(self.cache, os.path.basename(model_path))
        shutil.copy(model_path, path)
        self.basename = path
        self.basetype = ModelType.LORA_model
        return self.basename, self.basetype

class AutoModelLoader(BaseLoader):
    def __model_type_convert(self, model_path: str):
        def __logger(mt: ModelType) -> ModelType:
            print(f'The model was classified as \"{mt.value}\".')
            return mt
        modelType = super().determine_model_from_file(model_path)
        print('Broadly classify model types.')
        if modelType in [LoadModelType.SD_model_Ckpt,
                             LoadModelType.SD_model_Safetensors,
                             LoadModelType.SD_model_Diffusers,
                             LoadModelType.HuggingFace]:
            return __logger(ModelType.SD_model)
        elif modelType in [LoadModelType.VAE_model_Diffusers,
                             LoadModelType.VAE_model_Pt]:
            return __logger(ModelType.VAE_model)
        elif modelType == LoadModelType.LORA_model_Safetensors:
            return __logger(ModelType.LORA_model)
        raise ModelFormatError('The specified model could not be determined.')

    def set_model(self, model_path: str) -> (str, ModelType):
        model_type = self.__model_type_convert(model_path)
        print('Loading by model')
        if model_type == ModelType.SD_model:
            self.basename, self.basetype = DiffusionModelLoader(self.dtype, self.cache).set_model(model_path)
        elif model_type == ModelType.VAE_model:
            self.basename, self.basetype = VaeModelLoader(self.dtype, self.cache).set_model(model_path)
        elif model_type == ModelType.LORA_model:
            self.basename, self.basetype = LoraModelLoader(self.dtype, self.cache).set_model(model_path)
        else:
            raise ModelLoadError('Could not load model.')
        print(f'Model loading is complete. It is saved in \"{self.basename}\". The model type is \"{self.basetype.value}\"')
        return self.basename, self.basetype