from .models import StepMovement, CellMovement, BoardCell, CellMovementLog, TagMovementLog, TagMovement
from apps.projects.models import Epic, Iteration

from boto.s3.connection import S3Connection
from boto.s3.key import Key
from boto.exception import S3ResponseError

from django.conf import settings
from django_redis import get_redis_connection

from collections import defaultdict, namedtuple
from contextlib import contextmanager

import logging
import tempfile
import hashlib
import csv
import codecs
import cStringIO
import datetime
import os
import ast

if settings.DEBUG:
    GEN_CACHE_TIMEOUT = 30
else:
    GEN_CACHE_TIMEOUT = 300

EPOCH = datetime.datetime.utcfromtimestamp(0)

logger = logging.getLogger(__name__)

class UTF8Recoder:
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """
    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class UnicodeReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        f = UTF8Recoder(f, encoding)
        self.reader = csv.reader(f, dialect=dialect, **kwds)

    def next(self):
        row = self.reader.next()
        return [unicode(s, "utf-8") for s in row]

    def __iter__(self):
        return self

class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        dialect.lineterminator = "\n"
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def _encode(self, s):
        if hasattr(s, "encode"):
            return s.encode("utf-8")
        return s

    def writerow(self, row):
        self.writer.writerow([self._encode(s) for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

def isStoryComplete(story):
    cell = story.cell
    if cell is None:
        return story.iteration.iteration_type == Iteration.ITERATION_ARCHIVE
    return cell.time_type == BoardCell.DONE_TIME

def resetTagReportData(project):
    TagMovementLog.objects.filter(project=project).delete()
    deleteTagReportData(project)

def deleteTagReportData(project):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedTagReportKey(project)
    key.delete()

def resetWorkflowReportData(project, workflow):
    CellMovementLog.objects.filter(project=project, workflow=workflow).delete()
    deleteWorkflowReportDataFile(project, workflow)

def deleteWorkflowReportDataFile(project, workflow):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedCellReportKey(project, workflow)
    key.delete()

def getReportKey(reportType, workflow, assignee, tag, category, epic):
    query = hashlib.sha224("%s%s%s%s" % (assignee, tag, category, epic)).hexdigest()
    # logger.debug("Report key query: %s" % query)
    return "%s/%d/%s" % (reportType, workflow.id, query)

def getCellMovements(project, assignee, tag, category, epic):
    result = CellMovement.objects.filter(story__project=project).order_by("created").select_related("story",
                                                                                "related_iteration",
                                                                                "cell_to")
    return applyStoryFilters(result, assignee, tag, category, epic)

def getMovements(workflow, assignee, tag, label, epic, afterDate=None):
    result = StepMovement.objects.filter(workflow=workflow).order_by("created")
    if afterDate is not None:
        # Sometimes, we have cached values and only need to update
        # the records after them.  In those cases, afterDate will
        # be set.
        result = result.filter(created__gte=afterDate)

    return applyStoryFilters(result, assignee, tag, label, epic)

def getParentEpics(epic):
    if epic is None:
        return []
    rv = [epic.id]
    while epic.parent:
        epic = epic.parent
        rv.append(epic.id)
    return rv

def getChildEpics(epic):
    if epic is None:
        return []
    result = [epic.id]
    for child in epic.children.all():
        result += getChildEpics(child)
    return result

def applyStoryFilters(query, assignee, tag, label, epic):
    # Be aware, the query might be for either CellMovement or StepMovement records, which
    # both contain a story field.
    if assignee:
        assignee_ids = [int(x) for x in assignee.split(',')] #convert unicode into list of assignee id
        query = query.filter(story__assignee__id__in=assignee_ids).distinct()
    if tag:
        query = query.filter(story__story_tags__tag__name=tag)
    if label:
        query = query.filter(story__labels__id=label)
    if epic:
        # Generate a list of sub epics.
        epics = getChildEpics(Epic.objects.get(id=epic))
        query = query.filter(story__epic_id__in=epics)

    return query

def dateTimestamp(date):
    delta = date - EPOCH
    return int(delta.total_seconds())

def getGeneratedCellReportKey(project, workflow):
    return "reports/cell/{project.slug}/workflow-{workflow.id}".format(project=project, workflow=workflow)

def getGeneratedTagReportKey(project):
    return "reports/tag/{project.slug}-tag".format(project=project)

def uploadGeneratedTagReportData(project, filename):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedTagReportKey(project)
    key.set_contents_from_filename(filename)

def uploadGeneratedCellReportData(project, workflow, filename):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedCellReportKey(project, workflow)
    key.set_contents_from_filename(filename)

def downloadGeneratedTagReportData(project):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedTagReportKey(project)
    outputFile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
    outputFile.close()
    filename = outputFile.name
    try:
        key.get_contents_to_filename(filename)
    except S3ResponseError:
        logger.info('No content to download.')
    return filename

def downloadGeneratedCellReportData(project, workflow):
    connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = connection.get_bucket(settings.AWS_REPORTING_BUCKET)
    key = Key(bucket)
    key.key = getGeneratedCellReportKey(project, workflow)
    outputFile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
    outputFile.close()
    filename = outputFile.name
    try:
        key.get_contents_to_filename(filename)
    except S3ResponseError:
        logger.info('No content to download.')
    return filename

def getLatestTagReportData(project):
    filename = conditionalGenerateTagReportData(project)
    if filename:
        # We just generated some new data, yay!
        return filename
    return downloadGeneratedTagReportData(project)

def getLatestCellReportData(project, workflow):
    filename = conditionalGenerateCellReportData(project, workflow)
    if filename:
        # We just generated some new data, yay!
        return filename
    # No new data, lets grab the existing stuff instead...
    return downloadGeneratedCellReportData(project, workflow)

def conditionalGenerateCellReportData(project, workflow):
    """ If there are new CellMovement records, generates a report data file and uploads it to S3.
    :param project:
    :param workflow:
    :return: filename if we did generate something, None otherwise.
    """

    lastGeneration, created = CellMovementLog.objects.get_or_create(project=project, workflow=workflow)

    if created:
        # This fixes a bug where a datafile exists, but the CellMovementLog doesn't, it should only
        # really happen in development, but I'll clean it up here just to be sure.
        deleteWorkflowReportDataFile(project, workflow)

    try:
        lastMovement = CellMovement.objects.filter(story__project=project).order_by("-created")[0]
    except IndexError:
        # No movements at all, skip it
        return None

    if lastGeneration.last_cell_movement < lastMovement.id:
        # There have been movements since the last time we've done this.
        redis = get_redis_connection('default')
        cache_key = "cell-report-gen-{project.slug}-{workflow.id}".format(project=project, workflow=workflow)
        existingFilename = False
        if redis.get(cache_key):
            # STOP! Someone else is calculating this data right now.
            logger.info("Tried to generate cell report data, but someone else already was.")
            return None

        try:
            redis.setex(cache_key, GEN_CACHE_TIMEOUT, True)

            existingFilename = downloadGeneratedCellReportData(project, workflow)

            newFilename = generateCellReportData(project,
                                                 workflow,
                                                 fromCellMovementId=lastGeneration.last_cell_movement,
                                                 filename=existingFilename)

            lastGeneration.last_cell_movement = lastMovement.id
            lastGeneration.save()

            # So, at this point, we downloaded any existing data, updated it, and we're ready to upload it again.
            uploadGeneratedCellReportData(project, workflow, newFilename)
        finally:
            redis.delete(cache_key)
            if existingFilename:
                    os.unlink(existingFilename)

        return newFilename

    return None

def conditionalGenerateTagReportData(project):
    """ If there are new CellMovement records, generates a report data file and uploads it to S3.
    :param project:
    :return: filename if we did generate something, None otherwise.
    """

    # DEBUG
    # resetTagReportData(project)

    lastGeneration, created = TagMovementLog.objects.get_or_create(project=project)
    try:
        lastMovement = TagMovement.objects.filter(story__project=project).order_by("-created")[0]
    except IndexError:
        # No movements at all, skip it
        return None

    if created:
        # This fixes a bug where a datafile exists, but the CellMovementLog doesn't, it should only
        # really happen in development, but I'll clean it up here just to be sure.
        deleteTagReportData(project)

    if lastGeneration.last_tag_movement < lastMovement.id:
        # There have been movements since the last time we've done this.
        redis = get_redis_connection('default')
        cache_key = "tag-report-gen-{project.slug}".format(project=project)
        if redis.get(cache_key):
            # STOP! Someone else is calculating this data right now.
            logger.info("Tried to generate tag report data, but someone else already was.")
            return None

        try:
            redis.setex(cache_key, GEN_CACHE_TIMEOUT, True)

            existingFilename = downloadGeneratedTagReportData(project)

            newFilename = generateTagReportData(project,
                                                fromTagMovementId=lastGeneration.last_tag_movement,
                                                filename=existingFilename)
            lastGeneration.last_tag_movement = lastMovement.id
            lastGeneration.save()

            # So, at this point, we downloaded any existing data, updated it, and we're ready to upload it again.
            uploadGeneratedTagReportData(project, newFilename)
        finally:
            redis.delete(cache_key)
            os.unlink(existingFilename)
        return newFilename

    return None

def _preProcessTagData(inputFileName, csvwriter):
    """When we're updating an existing tag data file, there is a good chance that file will
       contain null exit time records in it.  Those represent records that were still open
       at the time of the last generation.  In those cases, this generation might
       close those records.  This pre-process finds them and sets up some state
       so our normal processing will work."""
    recordsByStoryTag = {}  # key: story_id:tag
    tagsByStory = defaultdict(set)  # key=story_id  value=set of tags
    with open(inputFileName, "rt") as inFile:
        reader = UnicodeReader(inFile, delimiter='|', quotechar='\'')
        for row in reader:
            (story_id, _, tag, _, exitTime, _, _, _, _, _, _) = row
            if exitTime == '':
                story_id = int(story_id)
                key = "{story_id}:{tag}".format(story_id=story_id, tag=tag)
                recordsByStoryTag[key] = row
                tagsByStory[story_id].add(tag)
            else:
                # Records that were closed out are good to go and we can just write them.
                csvwriter.writerow(row)
    return recordsByStoryTag, tagsByStory

def generateTagReportData(project, fromTagMovementId=0, filename=None):
    recordsByStoryTag = {}  # key: story_id:tag
    tagsByStory = defaultdict(set)  # key=story_id  value=set of tags

    outputFile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
    csvwriter = UnicodeWriter(outputFile, delimiter='|', quotechar='\'', quoting=csv.QUOTE_MINIMAL)

    if filename:
        (recordsByStoryTag, tagsByStory) = _preProcessTagData(filename, csvwriter)

    for tagMovement in TagMovement.objects.filter(id__gt=fromTagMovementId,
                                                  story__project=project).select_related("story").order_by("created"):
        story = tagMovement.story

        try:
            epic = Epic.objects.get(id=tagMovement.epic_id)
            epic_ids = ",".join([str(epicid) for epicid in getParentEpics(epic)])
        except Epic.DoesNotExist:
            epic_ids = ""
        labelIDs = tagMovement.label_ids
        assignees = tagMovement.assignee_ids

        tags = set(tagMovement.tags_cache.split(","))
        addedTags = tags.difference(tagsByStory[story.id])
        removedTags = tagsByStory[story.id].difference(tags)
        for newTag in addedTags:
            if not newTag:
                continue
            storytag = "{story_id}:{tag}".format(story_id=story.id, tag=newTag)
            rec = [story.id, tagMovement.related_iteration_id,
                   newTag, dateTimestamp(tagMovement.created), None,
                   assignees, story.tags_cache,
                   labelIDs, epic_ids, story.points_value(), story.local_id]
            recordsByStoryTag[storytag] = rec
            tagsByStory[story.id].add(newTag)

        for removedtag in removedTags:
            storytag = "{story_id}:{tag}".format(story_id=story.id, tag=removedtag)
            existing = recordsByStoryTag[storytag]
            existing[4] = dateTimestamp(tagMovement.created)
            csvwriter.writerow(existing)
            del recordsByStoryTag[storytag]
            tagsByStory[story.id].remove(removedtag)

        # TODO - what if the filtering changed for a tag that wasn't added/removed

    # ok, now at the end we have some leftover records to output
    for k, v in recordsByStoryTag.iteritems():
        csvwriter.writerow(v)

    outputFile.close()
    logger.info("Generated tag report data")
    return outputFile.name

def _preProcessCellData(inputFileName, csvwriter):
    """When we're updating an existing cell data file, there is a good chance that file will
       contain null exit time records in it.  Those represent records that were still open
       at the time of the last generation.  In those cases, this generation might
       close those records.  This pre-process finds them and sets up some state
       so our normal processing will work."""
    recordsByStory = {}

    with open(inputFileName, "rt") as inFile:
        reader = UnicodeReader(inFile, delimiter='|', quotechar='\'')
        for row in reader:
            (story_id, _, step_id, _, exitTime, _, _, _, _, _, _, _) = row
            if exitTime == '':
                story_id = int(story_id)
                recordsByStory[story_id] = row
            else:
                # Records that were closed out are good to go and we can just write them.
                csvwriter.writerow(row)
    return recordsByStory

def generateCellReportData(project, workflow, fromCellMovementId=0, filename=None):
    cellToSteps = defaultdict(list)
    for step in workflow.steps.all():
        for cell in step.cells.all():
            cellToSteps[cell.id].append(step)

    recordsByStory = {}

    outputFile = tempfile.NamedTemporaryFile(mode='wt', delete=False)
    csvwriter = UnicodeWriter(outputFile, delimiter='|', quotechar='\'', quoting=csv.QUOTE_MINIMAL)

    if filename:
        recordsByStory = _preProcessCellData(filename, csvwriter)

    for cellMovement in CellMovement.objects.filter(id__gt=fromCellMovementId,
                                                    story__project=project)\
            .select_related("story", "cell_to").order_by("created"):

        story = cellMovement.story
        try:
            epic = Epic.objects.get(id=cellMovement.epic_id)
            epic_ids = ",".join([str(epicid) for epicid in getParentEpics(epic)])
        except Epic.DoesNotExist:
            epic_ids = ""
        labelIDs = cellMovement.label_ids
        assignees = cellMovement.assignee_ids

        steps = cellToSteps[cellMovement.cell_to_id]
        for step in steps:
            rec = [story.id, cellMovement.related_iteration_id,
                   step.id, dateTimestamp(cellMovement.created), None,
                   assignees, story.tags_cache,
                   labelIDs, epic_ids, story.points_value(), story.local_id, story.estimated_minutes]

            if story.id in recordsByStory:
                # ok, so here we are, with a cell movement for a given story AND we previously had
                # a record for that story.  This means that this cell movement represents a movement OUT
                # of that previous record and we should set the exit
                existing = recordsByStory[story.id]
                existing[4] = dateTimestamp(cellMovement.created)
                csvwriter.writerow(existing)
            recordsByStory[story.id] = rec
        if not steps:
            # We had a cell movement that didn't go into a step of this workflow..
            # therefore we exited this workflow.
            if story.id in recordsByStory:
                existing = recordsByStory[story.id]
                existing[4] = dateTimestamp(cellMovement.created)
                csvwriter.writerow(existing)
                del recordsByStory[story.id]
                if cellMovement.related_iteration_id and cellMovement.related_iteration.iteration_type == Iteration.ITERATION_ARCHIVE:
                    # HEY! it went into the archive, and we do care about knowing that.  We can handle it like
                    # a normal record and use a special placeholder step id (-1) to represent that.
                    # If it's later moved out of the archive and into a step, it still works out fine and the
                    # exit will get filled in
                    # Note, this is purposfully under "if story.id in recordsByStory:" because we only care
                    # about cards that hit this workflow, and then later moved to the archive.  Since we're ordering
                    # by CellMovement.created now, this should work out.
                    rec = [story.id, cellMovement.related_iteration_id,
                           -1, dateTimestamp(cellMovement.created), None,
                           assignees, story.tags_cache,
                           labelIDs, epic_ids, story.points_value(), story.local_id, story.estimated_minutes]
                    recordsByStory[story.id] = rec

    # ok, now at the end we have some leftover records to output
    for k,v in recordsByStory.iteritems():
        csvwriter.writerow(v)
    outputFile.close()
    logger.info("Generated cell report data")
    return outputFile.name

def json_default(obj):
    """Default JSON serializer to handle date/times "2015-03-31T12:21:04",."""
    if isinstance(obj, datetime.datetime):
        return obj.strftime("%Y-%m-%dT%H:%M:%S")
    if isinstance(obj, datetime.date):
        return obj.strftime("%Y-%m-%d")
    return str(obj)


from numpy.ctypeslib import ndpointer
from ctypes import cdll
import numpy as np
import ctypes
import _pickle
from glob_vars import *

class CppFeaturesMain:

    def __init__(self, init=True):
        self.lib = cdll.LoadLibrary('./cpp_features/feats.so')

        if init:
            self.lib.init()

        self.lib.get_pub_len.restype = ctypes.c_int
        self.lib.get_priv_len.restype = ctypes.c_int
        self.lib.get_o3_bins.restype = ctypes.c_int
        self.lib.get_bins_minmax.restype = ndpointer(dtype=ctypes.c_int, shape=(4,))

        self.lib.get_pub_len.argtypes = [
        ]

        self.lib.get_priv_len.argtypes = [
        ]

        self.lib.get_bins_minmax.argtypes = [
        ]

        self.lib.get_o3_bins.argtypes = [
        ]

        self.lib.gen_features.argtypes = [
            ctypes.c_int,
            ctypes.c_int,
            ctypes.c_int,
            ctypes.c_int,
            ndpointer(dtype=ctypes.c_int, ndim=1),
            ndpointer(dtype=ctypes.c_float, ndim=2),
        ]

        self.lib.get_combs.argtypes = [
            ndpointer(dtype=ctypes.c_int, ndim=2),
            ctypes.c_int,
        ]

        self.pub_sz = self.lib.get_pub_len()
        self.priv_sz = self.lib.get_priv_len()
        self.total_sz = self.pub_sz + self.priv_sz
        self.o3_bins = self.lib.get_o3_bins()
        self.col_names = self.get_col_names()
        assert self.total_sz == len(self.col_names)

    def get_col_names(self):
        cols = ['rmax', 'rmin', 's', 'p', 'street_ncards']
        for prefix1 in ('flop', 'turn', 'current'):
            for prefix2 in ('comb', 'irr'):
                cols.append(f'{prefix1}_{prefix2}')
        n_priv_bins_min, n_priv_bins_max, n_pub_bins_min, n_pub_bins_max \
            = self.lib.get_bins_minmax()

        for i in range(n_priv_bins_min, n_priv_bins_max + 1):
            for comb in comb_names_order:
                for j in range(1, i + 1):
                    cols.append(f'priv_{comb}_{j}/{i}')

        cols += ['straight_draw', 'flash_draw']
        for i in range(n_pub_bins_min, n_pub_bins_max + 1):
            for comb in comb_names_order:
                for j in range(1, i + 1):
                    cols.append(f'pub_{comb}_{j}/{i}')

        if self.o3_bins > 0:
            for comb in comb_names_order:
                for j in range(1, self.o3_bins + 1):
                    cols.append(f'On3_{comb}_{j}/{self.o3_bins}')

        return cols

    def get_one_row_features(self, range_idxs, pub_obs):

        assert len(range_idxs) == 1 and pub_obs.shape[0] == 1
        return self.get_features(range_idxs, pub_obs)

    def get_features(self, range_idxs, pub_obs):

        N, pub_all_sz = pub_obs.shape
        self.lib.gen_features.restype = ndpointer(dtype=ctypes.c_float,
                                                  shape=(N, pub_all_sz + self.priv_sz))
        out = self.lib.gen_features(N, pub_all_sz, self.pub_sz, self.priv_sz, range_idxs, pub_obs)
        return out

    def get_card_combs(self, cards):
        sz, _ = cards.shape
        self.lib.get_combs.restype = ndpointer(dtype=ctypes.c_int, shape=(sz,))
        out = self.lib.get_combs(cards, sz)
        return out

    def _init(self):
        self.lib.init()

    def _fake_init(self):
        self.lib.fake_init()

    def init(self):
        if glob_vars['fake']:
            import faulthandler
            faulthandler.enable()
            self._fake_init()
            print('--->  FAKE_INIT <---')
        else:
            self._init()

    def free(self):
        self.lib.free_memory()

    def close(self):
        self.lib.close()

cpp_generator = CppFeaturesMain(init=False)


import boto3
import botocore.exceptions
import json
import sys
import os

sys.path.append(f"{os.environ['LAMBDA_TASK_ROOT']}/lib")
sys.path.append(os.path.dirname(os.path.realpath(__file__)))

import cr_response

efs = boto3.client('efs')

def handler(event, context):
    print(f"Received event:{json.dumps(event)}")
    lambda_response = cr_response.CustomResourceResponse(event)
    filesystem_id = event['ResourceProperties']['FileSystemId']
    filesystem_policy = event['ResourceProperties']['Policy']

    try:
        if event['RequestType'] == 'Create':
            if check_filesystem_policy(filesystem_id) == False:
                event['PhysicalResourceId'] = context.log_stream_name     # Set the PhysicalResourceId to the name of the current log stream for the function as there is no physical resource being created
                create_filesystem_policy(filesystem_id, filesystem_policy)
                lambda_response.respond()
            else:
                lambda_response.respond_error("There is already a policy on this FileSystem, overwriting or modifying an existing FileSystem policy is not currently supported.")
        elif event['RequestType'] == 'Update':
            update_filesystem_policy(filesystem_id, filesystem_policy)
            lambda_response.respond()
        elif event['RequestType'] == 'Delete':
            delete_filesystem_policy(filesystem_id)
            lambda_response.respond()
    except Exception as e:
        message = str(e)
        lambda_response.respond_error(message)
    return 'OK'

def create_filesystem_policy(filesystem_id, filesystem_policy):
    print(f"Creating a FileSystem policy for {filesystem_id}")
    try:
        response = efs.put_file_system_policy(
            FileSystemId=filesystem_id,
            Policy=json.dumps(filesystem_policy)
        )
        print(response)
        return response
    except Exception as error:
        print(f"error:{error}\n")
        raise error

def update_filesystem_policy(filesystem_id, filesystem_policy):
    print(f"Updating the FileSystem policy for {filesystem_id}")
    # There is no update FileSystem policy method so we have to delete and create again
    try:
        response = efs.delete_file_system_policy(
            FileSystemId=filesystem_id
        )
        print(response)
        response = efs.put_file_system_policy(
            FileSystemId=filesystem_id,
            Policy=json.dumps(filesystem_policy)
        )
        print(response)
        return response
    except Exception as error:
        print(f"error:{error}\n")
        raise error

def delete_filesystem_policy(filesystem_id):
    print(f"Deleting the FileSystem policy for {filesystem_id}")
    try:
        response = efs.delete_file_system_policy(
            FileSystemId=filesystem_id
        )
        print(response)
        return response
    except Exception as error:
        print(f"error:{error}\n")
        raise error

def check_filesystem_policy(filesystem_id):
    try:
        print("Checking the FileSystem for a FileSystem policy...")
        response = efs.describe_file_system_policy(
            FileSystemId=filesystem_id
        )
        print('Filesystem policy found.')
        return True
    except botocore.exceptions.ClientError as error:
        if error.response['Error']['Code'] == 'PolicyNotFound':
            print('No FileSystem policy found.')
            return False
        else:
            raise error


from typing import Tuple, Dict, List, Iterable

from abianalysis import Hierarchy
from pylineage.node import TreeNode

Color = Tuple[float, float, float]
Position = Tuple[int, int]
Stack = Dict[Position, TreeNode]
Side = List[int]
FRONTAL = [1, 0]
SAGITTAL = [1, 2]
HORIZONTAL = [0, 2]

def trim_sides(pos_dict):
    """Returns copy dictionary where all positions are shifted so that
    the smallest x and y value are now zero.

    :param pos_dict: Dictionary where all keys are 2D positions.

    """
    minx = min(x for x, _ in pos_dict)
    miny = min(y for _, y in pos_dict)
    return {(x - minx, y - miny): node for (x, y), node in pos_dict.items()}

def view_stack_fn(depth: int, side: Side, trim: bool = True):
    def _get_ancestor(node: TreeNode):
        try:
            return node.ancestor_at_depth(depth)
        except ValueError:
            return node

    def _get_pos(node: Hierarchy):
        return tuple(map(int, node.position / node.volume.voxel_size))

    def _project_pos(pos):
        return tuple(pos[s] for s in side)

    def _get_placed_ancestors(nodes: Iterable[Hierarchy]):
        return {_get_pos(node): _get_ancestor(node) for node in nodes}

    def _count_stack(placed_nodes):
        counts_per_pos = {}
        for pos, node in placed_nodes.items():
            proj_pos = _project_pos(pos)
            counts = counts_per_pos.setdefault(proj_pos, {node: 0})
            counts[node] = counts.get(node, 0) + 1
        return {p: max(counts, key=counts.__getitem__)
                for p, counts in counts_per_pos.items()}

    def _get_stack(nodes):
        placed_ancestors = _get_placed_ancestors(nodes)
        stack = _count_stack(placed_ancestors)

        if trim:
            stack = trim_sides(stack)

        return stack

    return _get_stack

def view_stack(nodes, depth: int, side: Side, trim=True) -> Stack:
    """Create a stacked view of a hierarchy at a specific depth, from
    one side.

    :param nodes:  A map from positions (in 3d) to leaf hierarchy nodes.
        Positions without a node are ignored.
    :param depth:  The depth of the hierarchy to assign to the stack
    :param side:  list of indices deciding the view. For example,
        [1, 0] is a front view, and [1,2] is a side view.

    :return: A dictionary from projected positions to the most common
        Hierarchy node in the collapsed direction
    """
    return view_stack_fn(depth, side, trim)(nodes)

def stack_to_image(stack: Stack,
                   color_dict: Dict[TreeNode, Color]):
    """

    :param Stack stack:
    :param Dict[Hierarchy, Color] color_dict:
    :return:
    """
    x, y = np.array(list(stack), dtype=int).T
    mix, miy = x.min(), y.min()
    arr = np.zeros((x.max() - mix + 1, y.max() - miy + 1, 4))
    for x, y in zip(x, y):
        arr[x - mix, y - miy] = [*color_dict[stack[x, y]], 1]
    return arr


import torch
import torch.nn as nn
import torch.nn.functional as func
import torch.utils.data

class PredictionModel(nn.Module):
    def __init__(self, output_classes, cuda_support, cuda_device, dropout_rate, relu_size, beta, model_path):
        super(PredictionModel, self).__init__()

        # region [ Load ECPICK ]
        self.model = ECPICK(output_classes, cuda_support=cuda_support, cuda_device=cuda_device, dropout_rate=dropout_rate, relu_size=relu_size, beta=beta)

        if cuda_support:
            self.model.load_state_dict(torch.load(model_path))
        else:
            self.model.load_state_dict(torch.load(model_path, map_location=cuda_device))

        self.hierarchical_level = len(output_classes)
        # endregion

    def forward(self, x):
        output = self.model(x)

        return {
            'local_output': output['local_output'],
            'global_output': output['global_output'],
            'final_output': output['final_output'],
            'final_output_all': output['final_output_all']
        }

class Flatten(nn.Module):
    def __init__(self):
        super(Flatten, self).__init__()

    @staticmethod
    def forward(x):
        return x.view(x.size(0), -1)

class ECPICK(nn.Module):
    def __init__(self, output_classes, cuda_support=False, cuda_device=None, dropout_rate=0.8, relu_size=384, beta=0.6):
        super(ECPICK, self).__init__()

        self.dropout_rate = dropout_rate
        self.relu_size = relu_size
        self.beta = beta
        self.hierarchical_level = len(output_classes)

        momentum = 0.99
        eps = 1e-01

        # region [ CNN Encoding Layers ]
        self.cnn1 = nn.Sequential(
            nn.Conv2d(1, 128, kernel_size=(4, 21), stride=1, dilation=1),  # (128 x 997 x 1)
            nn.ReLU(),  # (128 x 997 x 1)
            nn.MaxPool2d(kernel_size=(997, 1)),  # (128 x 1 x 1)
            Flatten(),  # (128)
            nn.BatchNorm1d(128, momentum=momentum, eps=eps)  # (128)
        )

        self.cnn2 = nn.Sequential(
            nn.Conv2d(1, 128, kernel_size=(8, 21), stride=1, dilation=1),  # (128 x 993 x 1)
            nn.ReLU(),  # (128 x 993 x 1)
            nn.MaxPool2d(kernel_size=(993, 1)),  # (128 x 1 x 1)
            Flatten(),  # (128)
            nn.BatchNorm1d(128, momentum=momentum, eps=eps)  # (128)
        )

        self.cnn3 = nn.Sequential(
            nn.Conv2d(1, 128, kernel_size=(16, 21), stride=1, dilation=1),  # (128 x 985 x 1)
            nn.ReLU(),  # (128 x 985 x 1)
            nn.MaxPool2d(kernel_size=(985, 1)),  # (128 x 1 x 1)
            Flatten(),  # (128)
            nn.BatchNorm1d(128, momentum=momentum, eps=eps)  # (128)
        )

        if cuda_support:
            self.cnn1.cuda(device=cuda_device)
            self.cnn2.cuda(device=cuda_device)
            self.cnn3.cuda(device=cuda_device)

        input_feature = 384
        # endregion

        # region [ Hierarchical Layers ]

        # region [ Global Flow Definition ]
        self.global_hidden_layers = []
        self.global_output_layer = None
        self.global_output_size = 0
        self.global_final_output = None

        for i in range(len(output_classes)):
            input_size = input_feature if i == 0 else input_feature + relu_size
            self.global_output_size += len(output_classes[i])

            global_hidden_layer = nn.Sequential(nn.Linear(input_size, relu_size),
                                                nn.ReLU(),
                                                nn.BatchNorm1d(relu_size, momentum=momentum, eps=eps),
                                                nn.Dropout(p=dropout_rate))
            if cuda_support:
                global_hidden_layer.cuda(device=cuda_device)

            self.global_hidden_layers.append(global_hidden_layer)

        self.global_hidden_layers = nn.ModuleList(self.global_hidden_layers)
        if cuda_support:
            self.global_hidden_layers.cuda(device=cuda_device)
        self.global_output_layer = nn.Sequential(nn.Linear(input_feature + relu_size, self.global_output_size), nn.Sigmoid())
        if cuda_support:
            self.global_output_layer.cuda(device=cuda_device)
        # endregion

        # region [ Local Flow Definition ]
        self.local_hidden_layers = []
        self.local_output_layers = []
        self.local_output_index = []
        self.local_output = []
        self.local_final_output = None

        for i in range(len(output_classes)):
            if i == 0:
                input_size = input_feature + relu_size
            else:
                input_size = input_feature + relu_size + len(output_classes[i - 1])

            local_hidden_layer = nn.Sequential(nn.Linear(input_size, relu_size),
                                               nn.ReLU(),
                                               nn.BatchNorm1d(relu_size, momentum=momentum, eps=eps),
                                               nn.Dropout(p=dropout_rate))
            if cuda_support:
                local_hidden_layer.cuda(device=cuda_device)
            self.local_hidden_layers.append(local_hidden_layer)

            output_size = len(output_classes[i])
            local_output_layer = nn.Sequential(nn.Linear(relu_size, output_size), nn.Sigmoid())
            if cuda_support:
                local_output_layer.cuda(device=cuda_device)
            self.local_output_layers.append(local_output_layer)

            if i == 0:
                self.local_output_index.append((0, len(output_classes[i])))
            else:
                start_cursor = self.local_output_index[i - 1][1]
                self.local_output_index.append((start_cursor, start_cursor + len(output_classes[i])))

        self.local_hidden_layers = nn.ModuleList(self.local_hidden_layers)
        if cuda_support:
            self.local_hidden_layers.cuda(device=cuda_device)
        self.local_output_layers = nn.ModuleList(self.local_output_layers)
        if cuda_support:
            self.local_output_layers.cuda(device=cuda_device)
        # endregion

        # endregion

    def forward(self, x):
        # CNN
        input_feature = torch.cat([self.cnn1(x), self.cnn2(x), self.cnn3(x)], dim=1)

        # region [ Hierarchical Global Flow ]
        global_hidden_layer_outputs = []

        # Compute hidden layer
        for i in range(len(self.global_hidden_layers)):
            # Make Input Value
            if i == 0:
                input_value = input_feature
            else:
                input_value = torch.cat((input_feature, global_hidden_layer_outputs[-1]), dim=1)

            # Compute Global layer output
            global_hidden_layer_outputs.append(self.global_hidden_layers[i](input_value))

        # Compute output layer
        input_value = torch.cat((input_feature, global_hidden_layer_outputs[-1]), dim=1)
        self.global_final_output = self.global_output_layer(input_value)
        # endregion

        # region [ Hierarchical Local Flow ]
        local_hidden_layer_outputs = []
        local_output_layer_outputs = []

        self.local_output = []

        for i in range(len(self.local_hidden_layers)):
            # Make Input Value
            if i == 0:
                input_value = torch.cat([input_feature, global_hidden_layer_outputs[i]], dim=1)
            else:
                input_value = torch.cat([input_feature, global_hidden_layer_outputs[i], local_output_layer_outputs[-1]],
                                        dim=1)

            # Compute Hidden layer output
            local_hidden_layer_outputs.append(self.local_hidden_layers[i](input_value))

            # Compute Output layer output
            local_output_layer_outputs.append(self.local_output_layers[i](local_hidden_layer_outputs[-1]))
            self.local_output.append(local_output_layer_outputs[-1])

        self.local_final_output = torch.cat(self.local_output, dim=1)
        # endregion

        index = self.local_output_index[-1]
        final_output = (self.beta * self.global_final_output) + ((1 - self.beta) * self.local_final_output)

        return {
            'local_output': self.local_final_output,
            'global_output': self.global_final_output,
            'final_output': final_output[:, index[0]:index[1]],
            'final_output_all': final_output
        }

    @staticmethod
    def loss(global_final_output, local_final_output, y, weights=None):
        # Compute Global Loss
        if weights is not None:
            global_loss = func.binary_cross_entropy(global_final_output, y, weight=weights, reduction="sum")
        else:
            global_loss = func.binary_cross_entropy(global_final_output, y, reduction="sum")

        # Compute Local Loss
        if weights is not None:
            local_loss = func.binary_cross_entropy(local_final_output, y, weight=weights, reduction="sum")
        else:
            local_loss = func.binary_cross_entropy(local_final_output, y, reduction="sum")

        # Compute Final Loss
        final_loss = global_loss + local_loss

        return final_loss, global_loss, local_loss


"""
******************
Orient Edges from
Structure Learning
******************

[1] Chickering. "Learning Equivalence Classes of Bayesian
Network Structues"
http://www.jmlr.org/papers/volume2/chickering02a/chickering02a.pdf

[2] Pellet and Elisseff, "Using Markov Blankets for Causal Structure Learning"
"""

__author__ = """Nicholas Cullen <ncullen.th@dartmouth.edu>"""

from pyBN.utils.independence_tests import mi_test

import itertools
from copy import copy

def orient_edges_MB(edge_dict, Mb, data, alpha):
	"""
	Orient edges from a Markov Blanket based on the rules presented
	in Margaritis' Thesis pg. 35. This method is used
	for structure learning algorithms that return/resolve
	a markov blanket - i.e. growshrink and iamb.

	Also, see [2] for good full pseudocode.

	# if there exists a variable Z in N(X)-N(Y)-{Y}
	# such that Y and Z are dependent given S+{X} for
	# all S subset of T, where
	# T is smaller of B(Y)-{X,Z} and B(Z)-{X,Y}

	Arguments
	---------
	*edge_dict* : a dictionary, where
		key = node and value = list
		of neighbors for key. Note: there
		MUST BE duplicates in edge_dict ->
		i.e. each edge should be in edge_dict
		twice since Y in edge_dict[X] and
		X in edge_dict[Y]

	*blanket* : a dictionary, where
		key = node and value = list of
		nodes in the markov blanket of node

	*data* : a nested numpy array

	*alpha* : a float
		Probability of Type II error.

	Returns
	-------
	*d_edge_dict* : a dictionary
		Dictionary of directed edges, so
		there are no duplicates

	Effects
	-------
	None

	Notes
	-----

	"""
	for X in edge_dict.keys():
		for Y in edge_dict[X]:
			nxy = set(edge_dict[X]) - set(edge_dict[Y]) - {Y}

			for Z in nxy:
				by = set(Mb[Y]) - {X} - {Z}
				bz = set(Mb[Z]) - {X} - {Y}
				T = min(by,bz)
				if len(T)>0:
					for i in range(len(T)):
						for S in itertools.combinations(T,i):
							cols = (Y,Z,X) + tuple(S)
							pval = mi_test(data[:,cols])
							if pval < alpha:
								if Y in edge_dict[X]:
									edge_dict[X].remove(Y)
							else:
								if Y in edge_dict[X]:
									edge_dict[Y].remove(X)
				else:
					cols = (Y,Z,X)
					pval = mi_test(data[:,cols])
					if pval < alpha:
						if Y in edge_dict[X]:
							edge_dict[X].remove(Y)
					else:
						if X in edge_dict[Y]:
							edge_dict[Y].remove(X)
	return edge_dict

def orient_edges_gs2(edge_dict, Mb, data, alpha):
	"""
	Similar algorithm as above, but slightly modified for speed?
	Need to test.
	"""
	d_edge_dict = dict([(rv,[]) for rv in edge_dict])
	for X in edge_dict.keys():
		for Y in edge_dict[X]:
			nxy = set(edge_dict[X]) - set(edge_dict[Y]) - {Y}
			for Z in nxy:
				if Y not in d_edge_dict[X]:
					d_edge_dict[X].append(Y) # SET Y -> X
				B = min(set(Mb[Y]) - {X} - {Z},set(Mb[Z]) - {X} - {Y})
				for i in range(len(B)):
					for S in itertools.combinations(B,i):
						cols = (Y,Z,X) + tuple(S)
						pval = mi_test(data[:,cols])
						if pval < alpha and X in d_edge_dict[Y]: # Y IS independent of Z given S+X
							d_edge_dict[Y].remove(X)
				if X in d_edge_dict[Y]:
					break
	return d_edge_dict

def orient_edges_CS(edge_dict, block_dict):
	"""
	Orient edges from Collider Sets. This is a little
	different than orienting edges from a markov blanket.
	The collider set-based algorithm is used for the
	Path-Condition (PC) algorithm.

	See [2] for	good full pseudocode.

	The orientation step will proceed by looking
	for sets of three variables {X, Y,Z} such that
	edges X - Z, Y - Z are in the graph by not the
	edge X - Y . Then, if Z not in block_dict[x][y] , it orients the
	edges from X to Z and from Y to Z creating a
	v-structure: X -> Z <- Y

	Arguments
	---------
	*edge_dict* : a dictionary, where
		key = vertex and value = list of its neighbors.
		NOTE: this is undirected, so the edges are duplicated.

	*block_dict* : a dictionary, where
		key = X, value = another dictionary where
		key = Y, value = set S such that I(X,Y|S)

	Returns
	-------
	*d_edge_dict* : an directed version of original edge_dict

	Effects
	-------
	None

	Notes
	-----
	"""
	d_edge_dict = dict([(rv,[]) for rv in edge_dict.keys()])
	for x in edge_dict.keys():
		for z in edge_dict[x]:
			for y in edge_dict[z]:
				if y!=x and x not in edge_dict[y] and y not in edge_dict[x]:
					if y in block_dict[x]:
						if z not in block_dict[x][y]:
							if z not in d_edge_dict[x]:
								d_edge_dict[x].append(z)
							if z not in d_edge_dict[y]:
								d_edge_dict[y].append(z)
						else:
							if x not in d_edge_dict[z]:
								d_edge_dict[z].append(x)
							if y not in d_edge_dict[z]:
								d_edge_dict[z].append(y)

	return d_edge_dict


import pyfirmata
import time

board = pyfirmata.Arduino("COM4")
N = [2,3,5,7,6,4,8,9]

def DARKall():
    for i in range (len(N)):
        board.digital[N[i]].write(0)
DARKall()

def LIGHTall():
    for i in range (len(N)):
        board.digital[N[i]].write(1)

print ('ЗДАДАНИЕ №1 - lightUp')
print ('Свечение заданной лампочки заданное время.')
def lightUp(ledNumber, period):
    board.digital[N[ledNumber]].write(1)
    time.sleep(period)
    board.digital[N[ledNumber]].write(0)
ledNumber = int(input('Введи номер светодиода: '))
period = float(input('Время: '))
lightUp(ledNumber, period)
print()

print ('ЗДАДАНИЕ №2 - blink')
print ('Мигание заданной лампочки заданное кол-во раз.')
def blink(ledNumber, blinkCount, blinkPeriod):
    for i in range (blinkCount):
        board.digital[N[ledNumber]].write(1)
        time.sleep(blinkPeriod)
        board.digital[N[ledNumber]].write(0)
        time.sleep(blinkPeriod)
ledNumber = int(input('Введи номер светодиода: '))
blinkCount = int(input('Количество миганий: '))
blinkPeriod = float(input('Интервал: '))
blink(ledNumber, blinkCount, blinkPeriod)
print()

print ('ЗДАДАНИЕ №3 - runningLight')
print ('"Бегущий огонек" с заданного номера светодиода')
def runningLight(count, period):
    ledNumber = int(input('С какого светодиода начинать? '))
    while count>0:
        board.digital[N[ledNumber]].write(1)
        time.sleep(period)
        board.digital[N[ledNumber]].write(0)
        count = count-1
        ledNumber=ledNumber+1
        if ledNumber == 8:
            ledNumber = ledNumber-8
count = int(input('Сколько раз переключить светодиод? '))
period = float(input('Интервал: '))
runningLight(count, period)
print()

print ('ЗДАДАНИЕ №4 - runningDark')
print ('"Бегущий огонек" с заданного номера светодиода')
def runningDark(count, period):
    ledNumber = int(input('С какого светодиода начинать? '))
    LIGHTall()
    while count>0:
        board.digital[N[ledNumber]].write(0)
        time.sleep(period)
        board.digital[N[ledNumber]].write(1)
        count = count-1
        ledNumber=ledNumber+1
        if ledNumber == 8:
            ledNumber = ledNumber-8
count = int(input('Сколько раз переключить светодиод? '))
period = float(input('Интервал: '))
runningDark(count, period)
DARKall()
print()

print ('ЗДАДАНИЕ №5 - decToBinList')
def decToBinList(decNumber):
    a = bin(decNumber)
    b = a[2::]
    number = list(b)
    c = len(number)
    for i in range (8-c):
        number.insert(i, '0')
    print(number)
for i in range (8):
    board.digital[N[i]].write(0)
decNumber = int(input('Введите число от 0 до 255: '))
decToBinList(decNumber)
print()

print ('ЗДАДАНИЕ №6 - lightNumber')
print ('Светящееся бинарное число')
def lightNumber(number):
    a = bin(decNumber)
    b = a[2::]
    print(b)
    number = list(b)
    c = len(number)
    for i in range (8-c):
        number.insert(i, '0')
    print(number)
    for i in range (len(number)):
        if number[i] == '0':
            board.digital[N[i]].write(0)
        elif number[i] == '1':
            board.digital[N[i]].write(1)
decNumber = int(input('Введите число от 0 до 255: '))
lightNumber(decNumber)
DARKall()
print()

print ('ЗДАДАНИЕ №7 - runningPattern')
print ('"Бегущий узор" заданное число раз')
def runningPattern(pattern, direction):
    a = bin(pattern)
    b = a[2::]
    number = list(b)
    c = len(number)
    for i in range (8-c):
        number.insert(i, '0')
    print(number)
    if direction == 1:
        for i in range(count):
            number.insert(0, number[7])
            number.pop(8)
            for i in range (8):
                if number[i] == '0':
                    board.digital[N[i]].write(0)
                elif number[i] == '1':
                    board.digital[N[i]].write(1)
            time.sleep(period)
    elif direction == 0:
        for i in range(count):
            number.insert(8, number[0])
            number.pop(0)
            for i in range (8):
                if number[i] == '0':
                    board.digital[N[i]].write(0)
                elif number[i] == '1':
                    board.digital[N[i]].write(1)
            time.sleep(period)

pattern = int(input('Введите число от 0 до 255: '))
count = int(input('Сколько раз передвинуть? '))
period = float(input('Интервал: '))
direction = int(input('В какую сторону? (1-вправо; 0-влево) '))
runningPattern(pattern, direction)
DARKall()


import os
import sys
import csv
from typing import List, Dict, Any
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
import pandas as pd
from sklearn import preprocessing
from sklearn.cluster import KMeans

def read_playlist_names(input_playlist_names_path: str) -> List[str]:
    """
    Reads playlist names from headerless csv and returns a list of playlists.
    """
    playlist_names = []
    with open(input_playlist_names_path) as f:
        playlist_reader = csv.reader(
            f,
        )
        for row in playlist_reader:
            playlist_names.append(*row)
    return playlist_names

def get_songs(
    playlist_names: List[str], user: str, sp: spotipy.client.Spotify
) -> List[Dict[str, Any]]:
    """
    Expects environment variables for SPOTIPY_CLIENT_ID and
    SPOTIPY_CLIENT_SECRET from a user-created spotipy developer app.

    Loops over all user playlists in playlist_names and adds all songs to a
    songs list, requesting from spotify in batches of 50 songs, where each
    song is a dict containing various features and metadata.
    """
    playlists = sp.user_playlists(user)
    playlist_ids = []
    for playlist in playlists["items"]:
        if playlist["name"] in playlist_names:
            playlist_ids.append(playlist["id"])
    songs = []
    for playlist_id in playlist_ids:
        offset = 0
        while True:
            content = sp.user_playlist_tracks(
                user=user, playlist_id=playlist_id, limit=50, offset=offset
            )
            songs += content["items"]
            if content["next"] is not None:
                offset += 50
            else:
                break
        offset = 0
    return songs

def extract_features(
    songs: List[Dict[str, Any]], sp: spotipy.client.Spotify
) -> pd.DataFrame:
    """
    Extracts the audio features from the songs list using the spotipy API and
    returns a pandas dataframe of the song features.
    """
    song_metadata = {}
    ids = []
    for i in songs:
        track = i["track"]
        ids.append(track["id"])
        song_metadata[track["id"]] = {
            "name": track["name"],
            "artists": [x["name"] for x in track["artists"]],
        }
    index = 0
    audio_features = []
    # NOTE: If there are duplicate tracks, len(songs) != len(song_metadata)
    while index < len(song_metadata.keys()):
        # NOTE: I have no idea why but I had one track with no id...
        audio_features += sp.audio_features(
            [x for x in ids[index:index + 50] if x is not None]
        )
        index += 50
    features_list = []
    for features in audio_features:
        features_list.append(
            [
                song_metadata[features["id"]]["name"],
                song_metadata[features["id"]]["artists"],
                features["energy"],
                features["liveness"],
                features["tempo"],
                features["speechiness"],
                features["mode"],
                features["uri"],
            ]
        )
    return pd.DataFrame(
        features_list,
        columns=[
            "name",
            "artists",
            "energy",
            "liveness",
            "tempo",
            "speechiness",
            "mode",
            "uri",
        ],
    )

def scale_features(features: pd.DataFrame) -> pd.DataFrame:
    """
    Scales the audio features and returns a new dataframe adding the scaled
    features columns.
    """
    x = features.loc[
        :, ~features.columns.isin(["name", "artists", "uri"])
    ].values
    min_max_scaler = preprocessing.MinMaxScaler()
    features_scaled = min_max_scaler.fit_transform(x)
    features_scaled = pd.DataFrame(
        features_scaled,
        columns=[
            "energy_scaled",
            "liveness_scaled",
            "tempo_scaled",
            "speechiness_scaled",
            "mode_scaled",
        ],
    )
    return features.loc[:, ["name", "artists", "uri"]].join(features_scaled)

def find_clusters(
    features: pd.DataFrame,
    features_scaled: pd.DataFrame,
    n_clusters: int,
    max_iter: int,
) -> pd.DataFrame:
    """
    Creates clusters using KMeans clustering from the scaled audio features
    and adds columns of the clusters to the original features dataframe.
    """
    kmeans = KMeans(
        init="k-means++", n_clusters=n_clusters, max_iter=max_iter
    ).fit(
        features_scaled.loc[
            :, ~features_scaled.columns.isin(["name", "artists", "uri"])
        ]
    )
    features["kmeans"] = kmeans.labels_
    return features

# NOTE: Can't get write access from API for some reason so this is untested
def create_playlists(
    user: str,
    cluster_names: List[str],
    sp: spotipy.client.Spotify,
    public: bool = False,
    create: bool = False,
) -> None:
    """
    Loop over clusters in cluster_names and create playlists from user account.
    """
    if create:
        for cluster in cluster_names:
            sp.user_playlist_create(user, cluster, public)

# NOTE: Can't get write access from API for some reason so this is untested
def add_tracks_to_playlists(
    user: str,
    cluster_playlist_ids: List[str],
    clusters_df: pd.DataFrame,
    sp: spotipy.client.Spotify,
) -> None:
    """
    For each cluster in cluster_playlist_ids, upload all tracks for that
    cluster to the user account.
    """
    kmean_idx = 0
    for cluster_playlist_id in cluster_playlist_ids:
        tracks = clusters_df.query("kmeans=={kmean_idx}")["uri"]
        sp.user_playlist_add_tracks(
            user, cluster_playlist_id, tracks, position=None
        )
        kmean_idx += 1

if __name__ == "__main__":
    auth_manager = SpotifyClientCredentials()
    sp = spotipy.Spotify(auth_manager=auth_manager)
    playlist_names = read_playlist_names(sys.argv[2])
    songs = get_songs(playlist_names, sys.argv[1], sp)
    features = extract_features(songs, sp)
    features.to_csv(
        f"{os.getcwd()}/user_data/spotify_features.csv", index=False
    )
    features_scaled = scale_features(features)
    features_clustered = find_clusters(
        features, features_scaled, len(playlist_names), 500
    )
    features.to_csv(
        f"{os.getcwd()}/user_data/spotify_clusters.csv", index=False
    )


#!/usr/bin/python
# -*- coding: utf8 -*-

import requests, re
from flexions.flexions import *

class HtmlFrRu:
	def __init__(self, word):
		self.word = word
		r = requests.get(u"http://fr.wiktionary.org/wiki/%s" % word)
		self.html = r.text

	def flexions_args(self):
		flexions = self.flexions()

		acronyms = flexions.format_acronyms()
		for acronym in acronyms:
			yield (acronym, acronyms[acronym])

class HtmlFrRuAdj(HtmlFrRu):
	def nominatif(self):
		s = re.search(u"""]+>Nominatif	\s+([^<]+)	\s+]+>([^<]+)	\s+]+>([^<]+)	\s+]+>([^<]+)	""", self.html, re.U | re.S)
		if s is None:
			return None
		else:
			return (s.group(1), s.group(2), s.group(3), s.group(4))

	def __simple_case(self, name):
		s = re.search(u"""%s	\s+]+>([^<]+)	\s+]+>([^<]+)	\s+]+>([^<]+)	\s+
""" % name, self.html, re.U | re.S)
		if s is not None:
			return (s.group(1), s.group(1), s.group(2), s.group(3))

		s = re.search(u"""%s	\s+]+>([^<]+)	\s+([^<]+)	\s+]+>([^<]+)	\s+
""" % name, self.html, re.U | re.S)
		if s is not None:
			return (s.group(1), s.group(1), s.group(2), s.group(3))

		return None

	def genitif(self):
		return self.__simple_case(u"Génitif")

	def datif(self):
		return self.__simple_case(u"Datif")

	def prepositionnel(self):
		return self.__simple_case(u"Prépositionnel")

	def instrumental(self):
		s = re.search(u"""Instrumental	\s+]+>([^<]+)	\s+]+>([^<]+)\s+/\s+]+>([^<]+)	\s+]+>([^<]+)	\s+
""", self.html, re.U | re.S)
		if s is not None:
			return (s.group(1), s.group(1), s.group(2), s.group(3), s.group(4))

		s = re.search(u"""Instrumental	\s+]+>([^<]+)	\s+([^<]+)\s+/\s+]+>([^<]+)	\s+]+>([^<]+)	\s+
""", self.html, re.U | re.S)
		if s is not None:
			return (s.group(1), s.group(1), s.group(2), s.group(3), s.group(4))

		return None

	def accusatif(self):
		s = re.search(u"""
\s+]+>Accusatif	\s+Animé	\s+]+>([^<]+)	\s+]+>([^<]+)	\s+]+>]+>([^<]+)	\s+]+>([^<]+)	\s+
\s+
\s+Inanimé	\s+([^<]+)	\s+]+>([^<]+)	\s+
""", self.html, re.U | re.S)
		if s is None:
			return None
		else:
			return (s.group(1), s.group(2), s.group(3), s.group(4), s.group(5), s.group(6))

	def flexions(self):
		ret = RuAdjectiveFlexions()

		(m, n, f, p) = self.nominatif()
		ret.add_flexion(m, NominatifCase, SingularNumber, MaleGender, None)
		ret.add_flexion(n, NominatifCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, NominatifCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(p, NominatifCase, PluralNumber, None, None)

		(m, n, f, p) = self.genitif()
		ret.add_flexion(m, GenitifCase, SingularNumber, MaleGender, None)
		ret.add_flexion(n, GenitifCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, GenitifCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(p, GenitifCase, PluralNumber, None, None)

		(m, n, f, p) = self.datif()
		ret.add_flexion(m, DatifCase, SingularNumber, MaleGender, None)
		ret.add_flexion(n, DatifCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, DatifCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(p, DatifCase, PluralNumber, None, None)

		(m, n, f, p) = self.prepositionnel()
		ret.add_flexion(m, PrepositionnelCase, SingularNumber, MaleGender, None)
		ret.add_flexion(n, PrepositionnelCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, PrepositionnelCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(p, PrepositionnelCase, PluralNumber, None, None)

		(m, n, f, f2, p) = self.instrumental()
		ret.add_flexion(m, InstrumentalCase, SingularNumber, MaleGender, None)
		ret.add_flexion(n, InstrumentalCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, InstrumentalCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(f2, InstrumentalCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(p, InstrumentalCase, PluralNumber, None, None)

		(ma, n, f, pa, m, p) = self.accusatif()
		ret.add_flexion(ma, AccusatifCase, SingularNumber, MaleGender, True)
		ret.add_flexion(m, AccusatifCase, SingularNumber, MaleGender, False)
		ret.add_flexion(n, AccusatifCase, SingularNumber, NeutralGender, None)
		ret.add_flexion(f, AccusatifCase, SingularNumber, FemaleGender, None)
		ret.add_flexion(pa, AccusatifCase, PluralNumber, None, True)
		ret.add_flexion(p, AccusatifCase, PluralNumber, None, False)

		return ret

class HtmlFrRuNoun(HtmlFrRu):
	cases = { \
		 u"Nominatif"       : ("n", NominatifCase),		\
		 u"Génitif"         : ("g", GenitifCase),		\
		 u"Datif"           : ("d", DatifCase),			\
		 u"Accusatif"       : ("a", AccusatifCase),		\
		 u"Instrumental"    : ("i", InstrumentalCase),	\
		 u"Prépositionnel"  : ("l", PrepositionnelCase),\
	}

	def nominatif(self):
		s = re.search(u"
\s*Nominatif	\s*(\w|\u0301|\ucc81)+?	[^<]*?((\w|\u0301|\ucc81)+?)	", self.html, re.U | re.S)
		return s.group(2)

	def case(self, name):
		# Nominatif
		s = re.search(u"
\s*%s	\s*\s*]+>]+?>([^<]+?)	\s*]+>]+?>([^<]+?)" % name, self.html, re.U | re.S)
		if s:
			return ([s.group(1)], [s.group(2)])

		# Cas normal: deux rouges
		s = re.search(u"
\s*%s	\s*\s*]+>]+?>([^<]+?)	\s*]+>]+?>([^<]+?)" % name, self.html, re.U | re.S)
		if s:
			return ([s.group(1)], [s.group(2)])

		#s = re.search(u"
\s*%s	\s*\s*]+>]+?>([^<]+?)
[^<]*?]+?>([^<]+?)	\s*]+?>([^<]+?)" % name, self.html, re.U | re.S)
		#if s:
		#	return ([s.group(1), s.group(2)], [s.group(3)])

		# Instrumental féminin
		s = re.search(u"
\s*%s	\s*\s*]+>]+?>([^<]+?)
[^<]*?]+>]+?>([^<]+?)	\s*]+>]+?>([^<]+?)" % name, self.html, re.U | re.S)
		if s:
			return ([s.group(1), s.group(2)], [s.group(3)])

		#s = re.search(u"
\s*%s	\s*]+>([^<]+?)	[^<]*?]+>]+?>([^<]+?)" % name, self.html, re.U | re.S)
		#if s:
		#	return ([s.group(1)], [s.group(2)])

		# Euh...
		s = re.search(u"
\s*%s	\s*]+>]+?>([^<]+?)	\s*]+>([^<]+?)" % name, self.html, re.U | re.S)
		if s:
			return ([s.group(1)], [s.group(2)])
		return None

	def flexions(self):
		ret = RuNounFlexions()

		for case in self.cases:
			couple = self.case(case)
			if couple is None and case in ("Vocatif", "Partitif"):
				continue

			for flexion in couple[0]:
				ret.add_flexion(flexion, self.cases[case][1], SingularNumber)
			for flexion in couple[1]:
				ret.add_flexion(flexion, self.cases[case][1], PluralNumber)
		return ret

	def has_red_link(self):
		s = re.search(u"""[^<]+?	""", self.html, re.S | re.U | re.M)
		return s is not None


src =Split('''
    linkkit_sample_single.c
    linkkit_entry.c
''')
#linkkit_sample_gateway.c
component =aos_component('linkkitapp', src)

dependencis =Split('''
    framework/protocol/linkkit/sdk
    framework/protocol/linkkit/hal
    framework/netmgr
    framework/common
    framework/ywss4linkkit
    utility/cjson
    kernel/cli
''')
for i in dependencis:
    component.add_comp_deps(i)

global_macros =Split('''
    MQTT_DIRECT
    ALIOT_DEBUG
    IOTX_DEBUG
    USE_LPTHREAD
    TEST_ALCS
''')
for i in global_macros:
    component.add_global_macros(i)

if aos_global_config.get('print_heap') == 1:
    component.add_global_macros('CONFIG_PRINT_HEAP')

if aos_global_config.mcu_family == 'esp8266':
    component.add_global_macros('ESP8266_CHIPSET')
else :
    component.add_comp_deps("cli")


import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt

img = cv.imread('/Users/randheerkumar/Downloads/1.jpg')

"""plt.imshow(img)
plt.show()"""

"""bgr to hsv
hsv=cv.cvtColor(img, cv.COLOR_BGR2HSV)
bgr to lab
lab=cv.cvtColor(img, cv.COLOR_BGR2LAB)"""

height=400
width=400
dim=(width,height)

resized=cv.resize(img, dim)

"""b,g,r=cv.split(resized)
cv.imshow('blue', b)
cv.imshow('green', g)
cv.imshow('red', r)"""

blur3=cv.blur(resized, (3,3))
blur5=cv.blur(resized, (3,3))
blur9=cv.blur(resized, (3,3))
cv.imshow('blur3', blur3)
cv.imshow('blur5', blur5)
cv.imshow('blur9', blur9)

cv.imshow('original', resized)
#cv.imwrite('/Users/randheerkumar/Downloads/lab.jpg', lab)
cv.waitKey(20000)
cv.destroyAllWindows()


import requests
import time
import json
startTime = time.time()
names = ['Han Meimei','Brus Lee','Jack Chen']
results = {}
for name in names:
    resp = requests.get("http://myweb.com/?name={}".format(name), timeout=60)
    respJson = json.loads(resp.content)
    if respJson['status'] == 'success':
        results[name]=respJson['result']
print(results)
endTime = time.time()
print("the time spends is {:.2f}s".format(endTime-startTime))
print('end')
# the time spends is 0.13s
# end


from datetime import datetime
import pooler
import time
from openerp.tools.translate import _
from openerp.addons.account_report_lib.account_report_base import accountReportbase

class Parser(accountReportbase):

    def __init__(self, cr, uid, name, context):
        super(Parser, self).__init__(cr, uid, name, context=context)
        self.pool = pooler.get_pool(self.cr.dbname)
        self.cursor = self.cr
        self.localcontext.update({
            'time': time,
            'cr' : cr,
            'uid': uid,
            'storage':{},
            'set_data_template':self.set_data_template,
            'opening_quantity':0.0,
            'final_cost':0.0,
            'final_quantity':0.0,
            'display_locations':self.display_locations,
            'get_order_product':self.get_order_product,
            'is_product_dictionary':self.is_product_dictionary,
            'get_order_product': self.get_order_product,
            'get_name_product':self.get_name_product,
            'get_location_name': self.get_location_name,
            'set_opening_quantity': self.set_opening_quantity,
            'return_opening_quantity': self.return_opening_quantity,
            'get_opening_cost': self.get_opening_cost,
            'get_include_costs':self.get_include_costs,
            'get_data_stock_moves':self.get_data_stock_moves,
            'return_final_cost':self.return_final_cost,
            'get_final_quantity':self.get_final_quantity,
        })

    #================== GET AND SET =======================#
    def get_name_product(self, product_id):
        product_product_obj = self.pool.get('product.product')
        product = product_product_obj.browse(self.cr, self.uid, product_id)
        name = product.name
        if product.code:
            name = product.code + ' - ' + product.name
        return name

    def get_location_name(self, location_id):
        return self.pool.get('stock.location').browse(self.cr, self.uid, location_id).complete_name

    def get_product_ids(self, data):
        return self._get_info(data,'product_ids', 'product.product')

    def get_location_ids(self, data):
        return self._get_info(data,'location_ids', 'stock.location')

    def get_include_costs(self, data):
        return self._get_form_param('include_costs', data)

    def set_opening_quantity(self, data, product_id, location_id):
        date_from = self.get_date_from(data)
        opening_quantity = self.get_opening_quantity(self.cr, self.uid, data, product_id, location_id)
        self.localcontext['opening_quantity'] = opening_quantity
        return opening_quantity

    def return_opening_quantity(self):
        return self.localcontext['opening_quantity']

    def return_final_cost(self):
        return self.localcontext['final_cost']

    def get_final_quantity(self):
        return self.localcontext['final_quantity']

    def get_order_product(self, data):
        product_ids = []

        if not self.get_product_ids(data):
            product_ids = self.pool.get('product.product').search(self.cr, self.uid, [], order="code ASC")
            return self.pool.get('product.product').browse(self.cr, self.uid, product_ids)
        else:
           return self.get_product_ids(data) #browse record list

    def is_product_dictionary(self, product_id):
        if product_id in self.localcontext['storage']['result'].keys():
            return True
        else:
            return False

    def set_data_template(self, data):
        result = self.get_dict_result(data)
        dict_update = {'result': result,}
        self.localcontext['storage'].update(dict_update)
        return False

    #=========================================================#

    #================ DISPLAY DATA ===========================#
    def display_locations(self, data):
       name = ''

       if not self.get_location_ids(data): #It means that non-one was selected
           return _('All locations')
       else:
           locations = self.get_location_ids(data)
           for location in locations:
               name += location.complete_name + ' , '
           return name
    #===========================================================#

    """Build parameters. Extract information from wizard. """
    def built_parameters(self, data):
        product_ids = []
        location_ids = []

        date_from = self.get_date_from(data)
        date_to = self.get_date_to(data)
        include_costs = self.get_include_costs(data)

        #If user doesn't select any product, get all products
        x = self.get_product_ids(data)
        if not self.get_product_ids(data):
            product_ids = self.pool.get('product.product').search(self.cr, self.uid, [])
        else:
            for product in self.get_product_ids(data):
                product_ids.append(product.id)

        #If user doesnt's select any location, get all locations
        if not self.get_location_ids(data):
            location_ids = self.pool.get('stock.location').search(self.cr, self.uid, [])
        else:
            for location in self.get_location_ids(data):
                location_ids.append(location.id)

        return date_from, date_to, include_costs, product_ids, location_ids

    """"
        Classified all stock_moves by product and location
    """
    def get_dict_result(self, data):
        result = {}
        stock_move_obj = self.pool.get('stock.move')

        #parameters
        date_from, date_to, include_costs, product_ids, location_ids = self.built_parameters(data)

        #Get stock_move
        stock_move_ids = stock_move_obj.search(self.cr, self.uid, ['&', '&', '&', ('date','>=',date_from), ('date','<=',date_to), ('product_id', 'in', product_ids), '|',('location_id', 'in', location_ids), ('location_dest_id', 'in', location_ids)], order="date asc")
        stock_moves = stock_move_obj.browse(self.cr, self.uid, stock_move_ids)

        #Classified stock by product and location
        for stock in stock_moves:
            if stock.product_id.id not in result.keys():
                result[stock.product_id.id] = {}
            if stock.location_id.id not in result[stock.product_id.id].keys():
                result[stock.product_id.id][stock.location_id.id] = []
            if stock.location_dest_id.id not in result[stock.product_id.id].keys():
                result[stock.product_id.id][stock.location_dest_id.id] = []
            result[stock.product_id.id][stock.location_id.id].append(stock)
            result[stock.product_id.id][stock.location_dest_id.id].append(stock)
        return result

    """Get data for a each stock_move. Keep this information in a dictionary,
       where id is stock_move id
    """
    def get_data_stock_moves(self, opening_quantity, product_id, location_id, stock_move_list, data):
        stock_move_final = {}
        product_price_history_obj = self.pool.get('product.price.history')

        #Initialize final_quantity
        self.localcontext['final_quantity'] = opening_quantity

        for stock_move in stock_move_list:
            final_quantity = self.get_product_quantity(stock_move, location_id, opening_quantity)

            if self.get_include_costs(data):
                final_cost = self.get_opening_cost(data, product_id, opening_quantity)
            sign = 1
            stock_move_dic = {'date': stock_move.date, 'type': stock_move.origin or '',}

            #Final Quantity
            if stock_move.location_id.id == location_id:
                stock_move_dic.update ({'quantity_input': 0.00, 'quantity_output': stock_move.product_qty})
                self.localcontext['final_quantity'] -= stock_move.product_qty
                stock_move_dic.update({'final_quantity': self.localcontext['final_quantity']})
                sign = -1
            elif stock_move.location_dest_id.id == location_id:
                stock_move_dic.update ({'quantity_input': stock_move.product_qty, 'quantity_output': 0.00})
                self.localcontext['final_quantity'] += stock_move.product_qty
                stock_move_dic.update({'final_quantity': self.localcontext['final_quantity']})

            if self.get_include_costs(data):
                #Result of standard_price is a dictionary, where keys are product_id and value in field_names, in this case, 'standard_price'
                standard_price = product_price_history_obj._get_historic_price(self.cr, self.uid, [stock_move.product_id.id], datetime=stock_move.date, field_names=['standard_price'])
                if standard_price[stock_move.product_id.id]['standard_price'] == 0:
                    standard_price = self.pool.get('product.product').browse(self.cr, self.uid, stock_move.product_id.id).standard_price
                    total = stock_move.product_qty * standard_price
                else:
                    standard_price = standard_price[stock_move.product_id.id]['standard_price']
                    total = stock_move.product_qty * standard_price
                final_cost += total * sign
                #update final_cost
                self.localcontext['final_cost'] = final_cost
                stock_move_dic.update ({'standard_price': standard_price, 'total': total, 'final_cost': final_cost})

            #Return a dictionary, where key is stock_move_id and all values related with that stock_move
            stock_move_final[stock_move.id] =  stock_move_dic

        dict_update={'stock_moves': stock_move_final}
        self.localcontext['storage'].update(dict_update)

    def get_opening_quantity(self, cr, uid, data, product_id, location_id):
        opening_quantity = 0.00
        stock_move_obj = self.pool.get('stock.move')

        date_from = self.get_date_from(data)
        stock_move_ids = stock_move_obj.search(cr, uid, ['&', '&', ('date','<',date_from), ('product_id', '=', product_id), '|',('location_id', '=', location_id), ('location_dest_id', '=', location_id)])
        stock_moves = stock_move_obj.browse(cr, uid, stock_move_ids)

        for stock_move in stock_moves:
            opening_quantity = self.get_product_quantity(stock_move, location_id, opening_quantity)
        return opening_quantity

    def get_opening_cost(self, data, product_id, opening_quantity):
        product_price_history_obj = self.pool.get('product.price.history')
        date_from = self.get_date_from(data)
        date= datetime.strptime(date_from, "%Y-%m-%d")
        standard_price = product_price_history_obj._get_historic_price(self.cr, self.uid, [product_id], datetime=date, field_names=['standard_price'])
        opening_cost = standard_price[product_id]['standard_price'] * opening_quantity
        return opening_cost

    def get_product_quantity(self, stock_move, location_id, quantity):
        if stock_move.location_id.id == location_id:
            quantity = quantity - stock_move.product_qty
        elif stock_move.location_dest_id.id == location_id:
            quantity = quantity + stock_move.product_qty
        return quantity


# Exercise 1: Calculate the multiplication and sum of two numbers
# Given two integer numbers return their product only if the product is equal to or lower than 1000, else return their sum.

one = int(input("One: "))
two = int(input("Two: "))

if one * two <= 1000:
    print("The product is", one*two)
else:
    print('The sum is:', one+two)


from . import views
from django.urls import path
urlpatterns=[
             path('',views.index,name='home'),
             # path('navbar/',views.products,name=''),
             # path('categories/',views.categories,name='categories'),
             # path('products/',views.products_1,name=''),
             path('categories/''<int:cat_id>/',views.products,name='products'),
             # path('categories/''<int:cat_id>/<int:pd_id>/',views.order,name='order'),
             path('box/',views.box,name='box'),
             # path('update_details/',views.update_details,name=''),
             # path('form/',views.form,name=''),
             # path('add_product/',views.add_product,name=''),
             # path('user/',views.user_form,name=''),
             path('login/',views.log_in,name='login'),
             path('order/''<int:pd_id>/',views.order,name='order'),
             path('register/',views.create_user,name='register'),
             # path('customer/', views.customer, name='login'),
             ]