import asyncio
from copy import deepcopy
import os

import pytest

import pbcoin.config as conf
from pbcoin.block import Block
from pbcoin.blockchain import BlockChain
from pbcoin.constants import TOTAL_NUMBER_CONNECTIONS
from pbcoin.mempool import Mempool
from pbcoin.mine import Mine
from pbcoin.netmessage import ConnectionCode, Errno, Message
from pbcoin.network import Node
from pbcoin.process_handler import ProcessingHandler
from pbcoin.trx import Trx
from pbcoin.utils.netbase import Addr
from pbcoin.wallet import Wallet

class TestNetworkBase:
    BASE_IP = '127.0.0'
    SENDER_IP = f'{BASE_IP}.1'
    RECEIVER_IP = f'{BASE_IP}.2'
    PORT = 8989

    @pytest.fixture
    async def run_nodes(self, request):
        """it runs n nodes that n come from request param also request has
        neighbor_finding for run or not start up for each node to find neighbors
        then save nodes and tasks (that is runners of nodes as async) in self.nodes
        and self.tasks Also, adds the close function of nodes in the finalizer
        that will be run after doing each test
        """
        self.__class__.nodes = []
        n, neighbor_finding = request.param
        for i in range(n):
            addr = Addr(ip=f"{self.BASE_IP}.{i+1}",
                        port=self.PORT,
                        pub_key=f"0x2{i+1}")  # TODO: make a valid public key with Key class
            blockchain = BlockChain([])
            unspent_coins = dict()
            wallet = Wallet(path_secret_key=os.environ["KEY_PATH"], unspent_coins=unspent_coins)
            mempool = Mempool()
            proc_handler = ProcessingHandler(blockchain, unspent_coins, wallet, mempool)
            node = Node(addr, proc_handler, 1)
            self.nodes.append(node)
        gathering = await asyncio.gather(*[
            node.create_a_server()
            for node in self.nodes
        ], return_exceptions=True)
        for f in gathering:
            if isinstance(f, Exception):
                raise f
        self.__class__.loops = []
        self.__class__.tasks = []
        for node in self.nodes:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            task = loop.create_task(node.listen())
            self.loops.append(loop)
            self.tasks.append(task)
            if not neighbor_finding or node.addr.ip == self.SENDER_IP:
                continue
            await node.start_up([f"{self.SENDER_IP}:{self.PORT}"], False)

        def close_nodes():
            for node in self.nodes:
                node.close()
            for loop in self.loops:
                loop.close()
            for task in self.tasks:
                task.cancel()
            self.tasks = []
        request.addfinalizer(close_nodes)

class TestMakeConnection(TestNetworkBase):
    @pytest.fixture
    def actual_neighbors(self):
        def actual(addr):
            data = dict()
            for node in self.nodes:
                if node.addr != addr:
                    data[node.addr.pub_key] = node.addr
            return data
        return actual

    @pytest.fixture
    def check_neighbors(self, actual_neighbors):
        def check():
            for node in self.nodes:
                assert len(node.neighbors) <= TOTAL_NUMBER_CONNECTIONS
                actual_len_neighbors = ((len(self.nodes) - 1)
                                        if (len(self.nodes)-1) <= TOTAL_NUMBER_CONNECTIONS
                                        else TOTAL_NUMBER_CONNECTIONS)
                assert len(node.neighbors) == actual_len_neighbors, f"{node.addr.hostname}"
                # TODO: better check neighbors when neighbors is more than CAPACITY
                assert node.neighbors.items() <= actual_neighbors(node.addr).items()
        return check

    @pytest.mark.parametrize(
        "run_nodes", [(2, False)], ids = ["two nodes"], indirect = True
    )
    async def test_is_up_and_interactable(self, run_nodes):
        """run 2 nodes and check they connected or not"""
        sender = self.nodes[0]
        receiver = self.nodes[1]
        assert sender.is_listening, "Sender is not listening"
        assert receiver.is_listening, "Receiver is not listening"
        assert await sender.send_ping_to(receiver.addr), "Could not make connection!"

    @pytest.mark.parametrize(
        "run_nodes", [(2, False)], ids = ["two nodes"], indirect = True
    )
    async def test_return_bad_message(self, run_nodes):
        """run 2 nodes and check they connected or not"""
        sender = self.nodes[0]
        receiver = self.nodes[1]
        message_res = await sender.connect_and_send(receiver.addr, b"Bluh", True)
        message_res = Message.from_str(message_res.decode())
        assert (not message_res.status) and (message_res.type_ == Errno.BAD_MESSAGE),  \
            "Other node hasn't understood that the message is wrong!"

    # Note: Handling the four neighbors' unittest seems to be handled and is ok.
    @pytest.mark.parametrize(
        "run_nodes",
        [(2, True), (4, True)],
        ids=["two nodes", "four nodes"],
        indirect=True
    )
    async def test_find_neighbors_more_total_connection_nodes(self,
                                                              run_nodes,
                                                              actual_neighbors):
        """run 2 and 4 nodes and check they handle neighbors even if more than
        total number connections for each nodes"""
        for node in self.nodes:
            assert len(node.neighbors) <= TOTAL_NUMBER_CONNECTIONS
            actual_len_neighbors = ((len(self.nodes) - 1)
                                    if (len(self.nodes)-1) <= TOTAL_NUMBER_CONNECTIONS
                                    else TOTAL_NUMBER_CONNECTIONS)
            assert len(node.neighbors) == actual_len_neighbors, f"{node.addr.hostname}"
            # TODO: better check neighbors when neighbors is more than TOTAL_NUMBER_CONNECTIONS
            assert node.neighbors.items() <= actual_neighbors(node.addr).items()

class TestProcessingHandler(TestNetworkBase):
    @pytest.fixture
    def mine_some_blocks(self):
        async def inner(n_mine: int, node: Node, send_mined_block=False, subsidies=None):
            """try to mine n blocks (n_mine) for the node and if send_mined_block is True
            then send minded blocks too. Also it returns blocks and if send_mined_block is
            True, returns errors too.
            """
            miner = Mine(node.proc_handler.blockchain,
                         node.proc_handler.wallet,
                         Mempool())
            blocks = []
            errors = []
            for n in range(n_mine):
                pre_hash = ""
                if n != 0:
                    pre_hash = blocks[-1].__hash__
                if subsidies is not None and len(subsidies) > n:
                    new_block = Block(pre_hash, n+1, subsidies[n])
                else:
                    new_block = Block(pre_hash, n+1)
                await miner.mine(setup_block=new_block,
                                 unspent_coins=node.proc_handler.unspent_coins,
                                 send_network=False)
                if send_mined_block:
                    errors += await node.send_mined_block(new_block)
                blocks.append(new_block)
            if send_mined_block:
                return blocks, errors
            else:
                return blocks
        return inner

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_mined_block(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        blocks, errors = await mine_some_blocks(1, sender, True)
        receiver = self.nodes[1]
        # assertions
        assert errors == [], "Mined block probably wrong"
        assert receiver.proc_handler.blockchain.height == 1,  \
               "Didn't received mined block"
        assert receiver.proc_handler.blockchain.last_block == blocks[-1],  \
               "Received block is not same with mined block"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_misplace_mined_block(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        blocks = await mine_some_blocks(2, sender, False)
        receiver = self.nodes[1]
        errors = await sender.send_mined_block(blocks[-1])
        # assertions
        assert errors == [], "Mined block probably wrong"
        assert receiver.proc_handler.blockchain.height == 2,  \
               "Didn't received mined block"
        assert receiver.proc_handler.blockchain.blocks == blocks,  \
               "Received block is not same with mined block"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_bad_mined_block(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        blocks = await mine_some_blocks(1, sender, False)
        blocks[0].block_hash = hex(conf.settings.glob.difficulty + 1)
        receiver = self.nodes[1]
        errors = await sender.send_mined_block(blocks[0])
        # assertion
        assert len(errors) != 0
        assert receiver.proc_handler.blockchain.height == 0, \
               "it received bad mined block"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_one_block_shorter_mined_block(self, run_nodes, mine_some_blocks):
        """Test handle new mine block which its height is less than other node blockchain length"""
        sender = self.nodes[0]
        sender_blocks = await mine_some_blocks(1, sender, False)
        receiver = self.nodes[1]
        receiver_blocks = await mine_some_blocks(2, receiver, False)
        errors = await sender.send_mined_block(sender_blocks[-1])
        # assertions
        assert errors == [], "Mined block probably wrong"
        assert sender.proc_handler.blockchain.height == 2,  \
               "Sender didn't received longer blockchain"
        assert sender.proc_handler.blockchain.blocks == receiver_blocks,  \
               "Sender blockchain is not same with receiver blockchain"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_two_blocks_shorter_mined_block(self, run_nodes, mine_some_blocks):
        """Test handle new mine block which its height is less than other node blockchain length"""
        sender = self.nodes[0]
        sender_blocks = await mine_some_blocks(1, sender, False)
        receiver = self.nodes[1]
        receiver_blocks = await mine_some_blocks(3, receiver, False)
        errors = await sender.send_mined_block(sender_blocks[-1])
        # assertions
        assert errors == [], "Mined block probably wrong"
        assert sender.proc_handler.blockchain.height == 3,  \
               "Sender didn't received longer blockchain"
        assert sender.proc_handler.blockchain.blocks == receiver_blocks,  \
               "Sender blockchain is not same with receiver blockchain"

    @pytest.mark.parametrize("run_nodes", [(2, False)], ids=["two nodes"], indirect=True)
    async def test_resolving_blockchain(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        receiver = self.nodes[1]
        sender_blocks = await mine_some_blocks(2, sender, False)
        _ = await mine_some_blocks(1, receiver, False)
        message = Message(True,
                          ConnectionCode.RESOLVE_BLOCKCHAIN,
                          receiver.addr,
                          {"blocks": [block.get_data() for block in sender_blocks]})
        rec = await sender.connect_and_send(receiver.addr,
                                            message.create_message(sender.addr),
                                            True)
        # assertions
        assert Message.from_str(rec.decode()).status, "Could not resolve"
        assert receiver.proc_handler.blockchain.height == 2, \
               "Didn't received new block or didn't resolve new blockchain"
        assert receiver.proc_handler.blockchain.last_block == sender_blocks[-1], \
               "Last Resolved block is not same with mined block"

    @pytest.mark.parametrize("run_nodes", [(2, False)], ids=["two nodes"], indirect=True)
    async def test_get_blocks_with_hash(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]  # send the blocks
        receiver = self.nodes[1]  # receive the blocks
        sender_blocks = await mine_some_blocks(2, sender, False)
        first_block = sender_blocks[0]
        message = Message(True,
                          ConnectionCode.GET_BLOCKS,
                          sender.addr,
                          {"hash_block": first_block.__hash__})
        response = await sender.connect_and_send(sender.addr,
                                                 message.create_message(receiver.addr),
                                                 True)
        response = Message.from_str(response.decode())
        # assertions
        assert (response.status is True
               and response.type_ == ConnectionCode.SEND_BLOCKS), \
               "Doesn't send proper message prototype"
        response_blocks = [Block.from_json_data_full(block)
                           for block in response.data["blocks"]]
        assert sender_blocks == response_blocks, "Didn't received correct blocks"

    @pytest.mark.parametrize("run_nodes", [(2, False)], ids=["two nodes"], indirect=True)
    async def test_get_blocks_with_bad_hash(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]  # send the blocks
        receiver = self.nodes[1]  # receive the blocks
        _ = await mine_some_blocks(2, sender, False)
        message = Message(True,
                          ConnectionCode.GET_BLOCKS,
                          sender.addr,
                          {"hash_block": "Bluh"})
        response = await sender.connect_and_send(sender.addr,
                                                 message.create_message(receiver.addr),
                                                 True)
        response = Message.from_str(response.decode())
        # assertions
        assert response.status is False \
               and response.type_ == Errno.BAD_BLOCK_VALIDATION, \
               "Sends the blocks for bad hash"

    @pytest.mark.parametrize("run_nodes", [(2, False)], ids=["two nodes"], indirect=True)
    async def test_get_blocks_with_index(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]  # send the blocks
        receiver = self.nodes[1]  # receive the blocks
        sender_blocks = await mine_some_blocks(2, sender, False)
        message = Message(True,
                          ConnectionCode.GET_BLOCKS,
                          sender.addr,
                          {"first_index": 0})
        response = await sender.connect_and_send(sender.addr,
                                                 message.create_message(receiver.addr),
                                                 True)
        response = Message.from_str(response.decode())
        # assertions
        assert (response.status is True
               and response.type_ == ConnectionCode.SEND_BLOCKS),  \
               "Doesn't send proper message prototype"
        response_blocks = [Block.from_json_data_full(block)
                           for block in response.data["blocks"]]
        assert sender_blocks == response_blocks, "Didn't received correct blocks"

    @pytest.mark.parametrize("run_nodes", [(2, False)], ids=["two nodes"], indirect=True)
    async def test_get_bad_blocks_with_index(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]  # send the blocks
        receiver = self.nodes[1]  # receive the blocks
        _ = await mine_some_blocks(2, sender, False)
        message = Message(True,
                          ConnectionCode.GET_BLOCKS,
                          sender.addr,
                          {"first_index": 1000})
        response = await sender.connect_and_send(sender.addr,
                                                 message.create_message(receiver.addr),
                                                 True)
        response = Message.from_str(response.decode())
        # assertions
        assert response.status is False \
               and response.type_ == Errno.BAD_BLOCK_VALIDATION, \
               "Sends the blocks for bad index"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_new_trx(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        receiver = self.nodes[1]
        wallet = sender.proc_handler.wallet
        blockchain = sender.proc_handler.blockchain
        subsidy = Trx(blockchain.height, wallet.public_key)
        await mine_some_blocks(1, sender, True, [subsidy])
        value = 25
        new_trx = None
        errors = []
        if value <= wallet.balance:
            new_trx = Trx.make_trx(sum(list(wallet.out_coins.values()), []),
                                   wallet.public_key, receiver.addr.pub_key, 25)
            mempool = sender.proc_handler.mempool
            result = mempool.add_new_transaction(new_trx,
                                                 wallet.sign(new_trx),
                                                 wallet.public_key,
                                                 sender.proc_handler.unspent_coins)
            assert result
            errors = await sender.send_new_trx(new_trx, wallet)
        assert errors == [], f"We have some errors in handling new trx: {errors}"
        assert receiver.proc_handler.mempool.is_exist(new_trx.hash_trx),  \
               "Didn't received new trx in mempool"

    @pytest.mark.parametrize("run_nodes", [(2, True)], ids=["two nodes"], indirect=True)
    async def test_handling_bad_new_trx(self, run_nodes, mine_some_blocks):
        sender = self.nodes[0]
        receiver = self.nodes[1]
        wallet = sender.proc_handler.wallet
        blockchain = sender.proc_handler.blockchain
        subsidy = Trx(blockchain.height, wallet.public_key)
        await mine_some_blocks(1, sender, True, [subsidy])
        value = 25
        new_trx = None
        errors = []
        if value <= wallet.balance:
            new_trx = Trx.make_trx(sum(list(wallet.out_coins.values()), []),
                                   wallet.public_key, receiver.addr.pub_key, 25)
            new_trx.inputs[0].value += 1  # corrupting
            mempool = sender.proc_handler.mempool
            result = mempool.add_new_transaction(new_trx,
                                                 wallet.sign(new_trx),
                                                 wallet.public_key,
                                                 sender.proc_handler.unspent_coins)
            assert not result, "Could not add new trx in mempool"
            errors = await sender.send_new_trx(new_trx, wallet)
        assert len(errors) != 0, "We haven't catch errors in handle new bad trx"
        assert receiver.proc_handler.mempool.is_exist(new_trx.hash_trx) is False,  \
               "Didn't received new trx in mempool"

# TODO: Write some comment and docs


# -*- coding: utf-8 -*-
from __future__ import absolute_import

import logging
import os
import random
import re
import string
import textwrap
from collections import OrderedDict
from copy import copy, deepcopy

import boto3
import kubernetes
from awacs.aws import Statement, Allow, Principal, Policy
from awacs.sts import AssumeRole
from botocore.exceptions import ClientError
from future.utils import viewitems, listvalues
from jinja2 import Environment
from kubernetes.client.rest import ApiException
from netaddr import IPNetwork
from troposphere import Tags, Template, Output, Ref, GetAtt, Join, Base64
from troposphere.autoscaling import AutoScalingGroup, Tag, LaunchConfiguration
from troposphere.ec2 import VPC, SecurityGroup, Subnet, SecurityGroupIngress, SecurityGroupEgress, InternetGateway, \
    VPCGatewayAttachment, RouteTable, Route, SubnetRouteTableAssociation, SecurityGroupRule
from troposphere.eks import Cluster, ResourcesVpcConfig
from troposphere.iam import Role, InstanceProfile
from troposphere.policies import UpdatePolicy, AutoScalingRollingUpdate

import ekscli
from ekscli import EKSCliException
from ekscli.utils import get_stack, ResourceReporter, Status, delete_stack, load_kubeconf

LOG = logging.getLogger(ekscli.__app_name__)

class AWSSecurityGroupRule:
    def __init__(self, cidr='0.0.0.0', protocol='tcp', from_port=-1, to_port=-1, port=None):
        self.cidr = cidr
        self.protocol = protocol
        if port:
            self.from_port = port
            self.to_port = port
        else:
            self.from_port = from_port
            self.to_port = to_port

class Resource:
    def __init__(self, name, description, status, resource_id=None):
        self.name = name
        self.description = description
        self.status = status
        self.resource_id = resource_id

class ClusterInfo:
    def __init__(self, name, endpoint=None, cert=None, vpc=None, subnets=[], sg=None):
        self.name = name
        self.endpoint = endpoint
        self.cert = cert
        self.vpc = vpc
        self.subnets = subnets
        self.sg = sg

class NodeGroupInfo:
    def __init__(self, name, instance=None, min_nodes=None, max_nodes=None, role=None):
        self.name = name
        self.instance = instance
        self.min = min_nodes
        self.max = max_nodes
        self.role = role

    def to_list(self):
        return [self.name, self.instance, self.min, self.max, self.role]

class ControlPlane:
    RESOURCE_EKS_VPC = Resource('VPC', 'VPC where the EKS runs on', Status.not_exist)
    RESOURCE_EKS_SUBNETS = []

    RESOURCE_CP_ROLE = Resource('MasterRole', 'EKS role for the control plane', Status.not_exist)
    RESOURCE_CP_SG = Resource('MasterSG', 'Security group for the control plane', Status.not_exist)
    RESOURCE_EKS_CLUSTER = Resource('Cluster', 'EKS cluster', Status.not_exist)

    # RESOURCE_VPC = 'VPC'
    RESOURCE_FORMAT_SUBNET = 'Subnet{}'
    RESOURCE_FORMAT_SUBNET_RTA = 'Subnet{}RouteTableAssociation'
    RESOURCE_VPC_INTERNET_GATEWAY = Resource('InternetGateway', 'VPC Internet gateway', Status.not_exist)
    RESOURCE_VPC_GATEWAY_ATTACHMENT = Resource('GatewayAttachment', 'VPC gateway attachment', Status.not_exist)
    RESOURCE_VPC_ROUTE_TABLE = Resource('VPCRouteTable', 'VPC Internet route table', Status.not_exist)
    RESOURCE_VPC_ROUTE = Resource('VPCRoute', 'VPC Internet route table', Status.not_exist)

    # Todo: use boto3.session.Session().get_available_regions('eks') once boto3 supports
    SUPPORTED_ZONES = {
        'us-east-1': ['us-east-1a', 'us-east-1b', 'us-east-1d'],
        'us-west-2': ['us-west-2a', 'us-west-2b', 'us-west-2c'],
    }
    SUPPORTED_KUBE_VERSIONS = ['1.10']

    OUTPUT_CP_ENDPOINT = 'MasterEndpoint'
    OUTPUT_CP_CA = 'MasterCA'
    OUTPUT_CP_SG = RESOURCE_CP_SG.name
    OUTPUT_VPC = RESOURCE_EKS_VPC.name
    OUTPUT_SUBNETS = 'EKSSubnets'

    CLUSTER_TAG_PATTERN = re.compile(r'kubernetes.io/cluster/(.*)')
    CLUSTER_TAG = 'kubernetes.io/cluster/{}'

    def __init__(self, name, role=None, subnets=None, region=None, kube_ver=None, tags=[],
                 vpc_cidr=None, zones=None):
        self.name = name
        self.stack_name = 'eks-{}-cp'.format(name)
        self.tag_name = 'eks.{}.cp'.format(name)
        self.subnets = subnets
        self.tags = dict(t.split('=') for t in tags)
        self.tags.update({'Name': self.tag_name, self.CLUSTER_TAG.format(self.name): 'owned'})
        self.vpc = None
        self.role = role
        self.region = region.lower() if region else boto3.session.Session().region_name
        self.kube_ver = kube_ver if kube_ver else '1.10'
        self.tpl = None
        self.vpc_cidr = vpc_cidr or '192.168.0.0/8'
        self.zones = zones or self.SUPPORTED_ZONES.get(self.region)
        self.resources = []
        self.subnet_refs = None

    def create(self):
        self._validate_creation()

        self.tpl = Template()
        self.tpl.add_version('2010-09-09')
        self.tpl.add_description('CFN template to create an EKS cluster and affiliated resources.')

        self._create_vpc()
        self._create_eks_control_plane()
        tags = [{'Key': k, 'Value': v} for (k, v) in viewitems(self.tags)]
        cf = boto3.session.Session().resource('cloudformation')
        stack = cf.create_stack(StackName=self.stack_name, TemplateBody=self.tpl.to_yaml(),
                                Capabilities=['CAPABILITY_NAMED_IAM'], Tags=tags)

        reporter = ResourceReporter()
        stack = reporter.report_stack_creation(self.stack_name, self.resources, stack.stack_id)

        return self._stack_to_cluster_info(self.name, stack.outputs)

    def delete(self):
        ci = self.query()
        if not ci:
            return

        stacks = self.get_all_stacks()
        for s in stacks:
            tags = {t.get('Key'): t.get('Value') for t in s.tags}
            if ControlPlane.CLUSTER_TAG.format(self.name) in tags and NodeGroup.NODEGROUP_TAG in tags:
                NodeGroup(name=tags.get(NodeGroup.NODEGROUP_TAG), cluster_info=ci).delete(stack=s)

        resources = deepcopy([self.RESOURCE_EKS_CLUSTER, self.RESOURCE_CP_SG, self.RESOURCE_CP_ROLE])
        for i, subnet in enumerate(ci.subnets):
            sname = self.RESOURCE_FORMAT_SUBNET.format(i + 1)
            resources.append(Resource(sname, 'EKS VPC {}'.format(sname), Status.created, subnet))
        vpc_resource = copy(self.RESOURCE_EKS_VPC)
        vpc_resource.resource_id = ci.vpc
        resources.append(vpc_resource)
        for r in resources:
            r.status = Status.created

        try:
            delete_stack(self.stack_name, resources)
        except EKSCliException as e:
            cf = boto3.session.Session().resource('cloudformation')
            stack = cf.Stack(self.stack_name)
            if not stack:
                raise e

            stack.delete()

    def query(self):
        cf = boto3.session.Session().resource('cloudformation')

        try:
            stack = cf.Stack(self.stack_name)
            return self._stack_to_cluster_info(self.name, stack.outputs)
        except ClientError as e:
            if 'Stack with id {} does not exist'.format(self.stack_name) in e.response.get('Error', {}).get('Message'):
                raise EKSCliException('The control plane of EKS cluster ({}) does not exist!'.format(self.name))

            raise EKSCliException(e)

    def get_all_stacks(self):
        cf = boto3.session.Session().resource('cloudformation')
        return [s for s in cf.stacks.all() for t in s.tags
                if t.get('Key') == self.CLUSTER_TAG.format(self.name) and t.get('Value') == 'owned']

    def get_all_nodegroup_stacks(self):
        return {t.get('Value'): s for s in self.get_all_stacks() for t in s.tags
                if t.get('Key') == NodeGroup.NODEGROUP_TAG}

    @staticmethod
    def _stack_to_cluster_info(name, outputs):
        if not outputs:
            return ClusterInfo(name)

        od = {o.get('OutputKey'): o.get('OutputValue') for o in outputs}
        return ClusterInfo(name, od.get(ControlPlane.OUTPUT_CP_ENDPOINT), od.get(ControlPlane.OUTPUT_CP_CA),
                           sg=od.get(ControlPlane.OUTPUT_CP_SG), vpc=od.get(ControlPlane.OUTPUT_VPC),
                           subnets=od.get(ControlPlane.OUTPUT_SUBNETS).split(','))

    def _validate_creation(self):
        stacks = get_stack(self.stack_name, ['DELETE_COMPLETE'])
        if len(stacks.get('unexpected')) > 0:
            raise EKSCliException(
                'CloudFormation stack (={}) of eks cluster already exists; delete it first.'.format(self.tag_name))

        if self.kube_ver not in self.SUPPORTED_KUBE_VERSIONS:
            raise EKSCliException('Error: Kubernetes Version (={}) is not supported in EKS'.format(self.kube_ver))

        if self.subnets:
            vpc_ids = self._get_vpc_ids_from_subnets(self.subnets)
            if len(set(vpc_ids)) != 1:
                raise EKSCliException('Subnets[{}] are not in the same vpc!'.format(','.join(self.subnets)))

            self.vpc = vpc_ids[0]
            self.RESOURCE_EKS_VPC.resource_id = self.vpc
            self.RESOURCE_EKS_VPC.status = Status.provided
            for i, sn in enumerate(self.subnets):
                r = Resource('Subnet{}'.format(i), description='Subnet{}'.format(i), status=Status.provided,
                             resource_id=sn)
                self.RESOURCE_EKS_SUBNETS.append(r)

    @staticmethod
    def _get_vpc_ids_from_subnets(subnets):
        resp = boto3.session.Session().client('ec2').describe_subnets(SubnetIds=subnets)
        return [subnet.get('VpcId') for subnet in resp.get('Subnets', [])]

    def _create_vpc(self):
        if self.vpc:
            self.tpl.add_output(Output(self.OUTPUT_VPC, Value=self.vpc))
            self.tpl.add_output(Output(self.OUTPUT_SUBNETS, Value=','.join(self.subnets)))
            return

        vpc = VPC(self.RESOURCE_EKS_VPC.name, CidrBlock=self.vpc_cidr, Tags=Tags(Name=self.tag_name))
        self.tpl.add_resource(vpc)
        gateway = self.tpl.add_resource(InternetGateway(self.RESOURCE_VPC_INTERNET_GATEWAY.name))
        self.tpl.add_resource(VPCGatewayAttachment(
            self.RESOURCE_VPC_GATEWAY_ATTACHMENT.name, VpcId=Ref(vpc), InternetGatewayId=Ref(gateway),
            DependsOn=gateway,
        ))
        rt = self.tpl.add_resource(RouteTable(
            self.RESOURCE_VPC_ROUTE_TABLE.name, VpcId=Ref(vpc), DependsOn=gateway,
            Tags=Tags(Name='public subnet', Network='public'),
        ))
        self.tpl.add_resource(Route(
            self.RESOURCE_VPC_ROUTE.name, RouteTableId=Ref(rt), DestinationCidrBlock='0.0.0.0/0',
            GatewayId=Ref(gateway),
        ))
        self.resources.extend(deepcopy([self.RESOURCE_EKS_VPC, self.RESOURCE_VPC_INTERNET_GATEWAY,
                                        self.RESOURCE_VPC_GATEWAY_ATTACHMENT, self.RESOURCE_VPC_ROUTE_TABLE,
                                        self.RESOURCE_VPC_ROUTE]))

        subnets = []
        vpc_network = IPNetwork(self.vpc_cidr)
        prefixlen = IPNetwork(self.vpc_cidr).prefixlen + (len(self.zones) - 1).bit_length()
        cidrs = list(vpc_network.subnet(prefixlen))
        for i, zone in enumerate(self.zones):
            sname = self.RESOURCE_FORMAT_SUBNET.format(i + 1)
            staname = self.RESOURCE_FORMAT_SUBNET_RTA.format(i + 1)
            subnet = self.tpl.add_resource(Subnet(
                sname, AvailabilityZone=zone, VpcId=Ref(vpc), CidrBlock=str(cidrs[i].cidr),
                Tags=Tags(Name='{}-{}'.format(self.name, str(i + 1)))
            ))
            self.resources.append(Resource(sname, 'EKS VPC {}'.format(sname), Status.not_exist))
            self.tpl.add_resource(SubnetRouteTableAssociation(
                staname, SubnetId=Ref(subnet), RouteTableId=Ref(rt)
            ))
            self.resources.append(Resource(staname, 'EKS VPC {}'.format(staname), Status.not_exist))
            subnets.append(subnet)

        self.subnet_refs = [Ref(s) for s in subnets]
        self.tpl.add_output(Output(self.OUTPUT_VPC, Value=Ref(vpc)))
        self.tpl.add_output(Output(self.OUTPUT_SUBNETS, Value=Join(',', self.subnet_refs)))

    def _create_eks_control_plane(self):
        account_id = boto3.client('sts').get_caller_identity().get('Account')
        r = copy(self.RESOURCE_CP_ROLE)
        if self.role:
            role_arn = 'arn:aws:iam::{}:role/{}'.format(account_id, self.role)
            r.status = Status.provided
            r.resource_id = role_arn
        else:
            role = self.tpl.add_resource(
                Role(self.RESOURCE_CP_ROLE.name, RoleName=self.tag_name,
                     AssumeRolePolicyDocument=Policy(Statement=[
                         Statement(Effect=Allow, Action=[AssumeRole],
                                   Principal=Principal('Service', ['eks.amazonaws.com'])),
                         Statement(Effect=Allow, Action=[AssumeRole],
                                   Principal=Principal('AWS', ['arn:aws:iam::{}:root'.format(account_id)])), ], ),
                     ManagedPolicyArns=['arn:aws:iam::aws:policy/AmazonEKSClusterPolicy',
                                        'arn:aws:iam::aws:policy/AmazonEKSServicePolicy']
                     ))
            role_arn = GetAtt(role, 'Arn')

        cp_sg = self.tpl.add_resource(SecurityGroup(
            self.RESOURCE_CP_SG.name,
            GroupDescription='Security Group applied to EKS cluster',
            VpcId=self.tpl.outputs.get(self.OUTPUT_VPC).resource.get('Value'),
            Tags=Tags(Name=self.tag_name)
        ))
        self.tpl.add_output(Output(self.OUTPUT_CP_SG, Value=Ref(self.OUTPUT_CP_SG)))

        self._create_eks_cluster_template(cp_sg, role_arn)
        self.resources.extend([r, copy(self.RESOURCE_CP_SG), copy(self.RESOURCE_EKS_CLUSTER)])

    # ToDo: remove this once moto support EKS in mock_cloudformation
    def _create_eks_cluster_template(self, cp_sg, role_arn):
        cluster = Cluster(self.RESOURCE_EKS_CLUSTER.name, Name=self.name, Version=self.kube_ver, RoleArn=role_arn,
                          ResourcesVpcConfig=ResourcesVpcConfig(SecurityGroupIds=[Ref(cp_sg)],
                                                                SubnetIds=self.subnets or self.subnet_refs))
        self.tpl.add_resource(cluster)
        self.tpl.add_output(Output(self.OUTPUT_CP_ENDPOINT, Value=GetAtt(cluster, 'Endpoint')))
        self.tpl.add_output(Output(self.OUTPUT_CP_CA, Value=GetAtt(cluster, 'CertificateAuthorityData')))

class KubeConfig:
    KUBE_CONFIG_YAML = textwrap.dedent("""\
    apiVersion: v1
    clusters:
    - cluster:
        server: {{ ci.endpoint }}
        certificate-authority-data: {{ ci.cert }}
      name: {{ ci.name }}
    contexts:
    - context:
        cluster: {{ ci.name }}
        user: {{ user | default('aws', True) }}
      name: {{ ci.name }}
    current-context: {{ ci.name }}
    kind: Config
    preferences: {}
    users:
    - name: {{ user | default('aws', True) }}
      user:
        exec:
          apiVersion: client.authentication.k8s.io/v1alpha1
          command: {{ heptio | default('heptio-authenticator-aws', True) }}
          args:
            - "token"
            - "-i"
            - "{{ ci.name }}"
    """)

    def __init__(self, cluster_info, kubeconf=None, user='aws', heptio_auth='heptio-authenticator-aws'):
        self.cluster_info = cluster_info
        self.user = user
        self.heptio = heptio_auth
        if kubeconf:
            self.kubeconf = kubeconf
        else:
            files = os.environ.get('KUBECONFIG', '~/.kube/config')
            self.kubeconf = os.path.expanduser(files.split(':')[0])

    def create(self):
        reporter = ResourceReporter()
        resource = Resource('kubeconf', 'Kubernetes configuration file', Status.not_exist, resource_id=self.kubeconf)
        reporter.progress(resource)
        try:
            if os.path.isfile(self.kubeconf):
                import oyaml as yaml
                with open(self.kubeconf, 'r') as cf:
                    kc = yaml.load(cf)

                clusters = self._get_components(kc, 'clusters')
                cs = [c for c in clusters if c.get('name') == self.cluster_info.name]
                if not cs:
                    clusters.append(OrderedDict([
                        ('cluster', OrderedDict([
                            ('certificate-authority-data', self.cluster_info.cert),
                            ('server', self.cluster_info.endpoint),
                        ])),
                        ('name', self.cluster_info.name),
                    ]))
                else:
                    for c in cs:
                        c['cluster']['server'] = self.cluster_info.endpoint
                        c['cluster']['certificate-authority-data'] = self.cluster_info.cert

                users = self._get_components(kc, 'users')
                us = [u for u in users if u.get('name') == self.user]
                if not us:
                    users.append(OrderedDict([
                        ('name', self.user),
                        ('user', OrderedDict([
                            ('exec', OrderedDict([
                                ('apiVersion', 'client.authentication.k8s.io/v1alpha1'),
                                ('command', self.heptio),
                                ('args', ['token', '-i', self.cluster_info.name])
                            ]))]))]))
                else:
                    for u in users:
                        u['user'] = OrderedDict([
                            ('exec', OrderedDict([
                                ('apiVersion', 'client.authentication.k8s.io/v1alpha1'),
                                ('command', self.heptio),
                                ('args', ['token', '-i', self.cluster_info.name])
                            ]))])

                contexts = self._get_components(kc, 'contexts')
                cs = [c for c in contexts if c.get('context', {}).get('cluster') == self.cluster_info.name
                      and c.get('context', {}).get('user') == self.user]
                if not cs:
                    contexts.append(OrderedDict([
                        ('context', OrderedDict([
                            ('cluster', self.cluster_info.name),
                            ('namespace', 'default'),
                            ('user', self.user),
                        ])),
                        ('name', self.cluster_info.name),
                    ]))

                kc['current-context'] = self.cluster_info.name

                with open(self.kubeconf, 'w') as cf:
                    cf.write(yaml.safe_dump(kc, default_flow_style=False))
            else:
                s = Environment().from_string(KubeConfig.KUBE_CONFIG_YAML).render(ci=self.cluster_info, user=self.user,
                                                                                heptio=self.heptio)
                with open(self.kubeconf, 'w') as cf:
                    cf.write(s)

            resource.status = Status.created
            resource.resource_id = self.kubeconf
            reporter.succeed(resource)
        except Exception as e:
            resource.status = Status.failed
            reporter.fail(resource)
            raise EKSCliException(e)

        return

    @staticmethod
    def _get_components(kubeconf, components):
        if components not in kubeconf:
            kubeconf[components] = []

        return kubeconf.get(components)

    def _validate(self):
        pass

class NodeGroup:
    RESOURCE_NG_KEYPAIR = Resource('KeyName', 'Imported keypair name', Status.not_exist)
    RESOURCE_NG_ROLE = Resource('NodeGroupRole', 'Node EC2 instance role', Status.not_exist)
    RESOURCE_NG_PROFILE = Resource('NodeGroupProfile', 'Node EC2 instance profile', Status.not_exist)
    RESOURCE_NG_SG = Resource('NodeSG', 'Node EC2 security group', Status.not_exist)
    RESOURCE_NG_SG_INGRESS = Resource('NodeSGIngress', 'Node EC2 security group intra-nodes ingress',
                                      Status.not_exist)
    RESOURCE_NG_SG_CP_INGRESS = Resource('NodeSGFromCPIngress', 'Node EC2 security group control plane ingress',
                                         Status.not_exist)
    RESOURCE_CP_EGRESS_TO_NG = Resource('CPEgressToNodeSG',
                                        'Control plane security group egress to node security group',
                                        Status.not_exist)
    RESOURCE_CP_SG_INGRESS = Resource('CPIngressToNodeSG', 'Control plane security group ingress for pods',
                                      Status.not_exist)
    RESOURCE_NG_ASG = Resource('NodeAutoScalingGroup', 'Node autoscaling group', Status.not_exist)
    RESOURCE_NG_ASG_LC = Resource('NodeLaunchConfig', 'Node autoscaling group launch configuration',
                                  Status.not_exist)

    OUTPUT_KEYNAME = 'KeyName'

    NODEGROUP_TAG = 'k8s.io/cluster-autoscaler/node-template/label/ng'

    DEFAULT_AMI = {'us-east-1': 'ami-dea4d5a1', 'us-west-2': 'ami-73a6e20b'}
    USER_DATA = textwrap.dedent('''\
    #!/bin/bash -xe
    CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki
    CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt
    MODEL_DIRECTORY_PATH=~/.aws/eks
    MODEL_FILE_PATH=$MODEL_DIRECTORY_PATH/eks-2017-11-01.normal.json
    mkdir -p $CA_CERTIFICATE_DIRECTORY
    mkdir -p $MODEL_DIRECTORY_PATH
    curl -o $MODEL_FILE_PATH https://s3-us-west-2.amazonaws.com/amazon-eks/1.10.3/2018-06-05/eks-2017-11-01.normal.json
    aws configure add-model --service-model file://$MODEL_FILE_PATH --service-name eks
    aws eks describe-cluster --region={{region}} --name={{ci.name}} --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > /tmp/describe_cluster_result.json
    cat /tmp/describe_cluster_result.json | grep certificateAuthorityData | awk '{print $2}' | sed 's/[,"]//g' | base64 -d > $CA_CERTIFICATE_FILE_PATH
    MASTER_ENDPOINT=$(cat /tmp/describe_cluster_result.json | grep endpoint | awk '{print $2}' | sed 's/[,"]//g')
    INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
    sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /var/lib/kubelet/kubeconfig
    sed -i s,CLUSTER_NAME,{{ci.name}},g /var/lib/kubelet/kubeconfig
    sed -i s,REGION,{{region}},g /etc/systemd/system/kubelet.service
    sed -i s,MAX_PODS,{{max_pods}},g /etc/systemd/system/kubelet.service
    sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /etc/systemd/system/kubelet.service
    sed -i s,INTERNAL_IP,$INTERNAL_IP,g /etc/systemd/system/kubelet.service
    DNS_CLUSTER_IP=10.100.0.10
    if [[ $INTERNAL_IP == 10.* ]] ; then DNS_CLUSTER_IP=172.20.0.10; fi
    sed -i s,DNS_CLUSTER_IP,$DNS_CLUSTER_IP,g  /etc/systemd/system/kubelet.service
    sed -i s,CERTIFICATE_AUTHORITY_FILE,$CA_CERTIFICATE_FILE_PATH,g /var/lib/kubelet/kubeconfig
    sed -i s,CLIENT_CA_FILE,$CA_CERTIFICATE_FILE_PATH,g  /etc/systemd/system/kubelet.service
    systemctl daemon-reload
    systemctl restart kubelet
    /opt/aws/bin/cfn-signal -e $? --stack {{stack_name}} --resource {{ ng_asg }} --region {{region}}
    ''')

    MAP_ROLE = textwrap.dedent('''\
    - rolearn: {{ role }}
      username: system:node:{% raw %}{{EC2PrivateDNSName}}{% endraw %}
      groups:
        - system:bootstrappers
        - system:nodes
    ''')

    MAX_PODS = {
        'c4.large': 29, 'c4.xlarge': 58, 'c4.2xlarge': 58, 'c4.4xlarge': 234, 'c4.8xlarge': 234, 'c5.large': 29,
        'c5.xlarge': 58, 'c5.2xlarge': 58, 'c5.4xlarge': 234, 'c5.9xlarge': 234, 'c5.18xlarge': 737, 'i3.large': 29,
        'i3.xlarge': 58, 'i3.2xlarge': 58, 'i3.4xlarge': 234, 'i3.8xlarge': 234, 'i3.16xlarge': 737, 'm3.medium': 12,
        'm3.large': 29, 'm3.xlarge': 58, 'm3.2xlarge': 118, 'm4.large': 20, 'm4.xlarge': 58, 'm4.2xlarge': 58,
        'm4.4xlarge': 234, 'm4.10xlarge': 234, 'm5.large': 29, 'm5.xlarge': 58, 'm5.2xlarge': 58, 'm5.4xlarge': 234,
        'm5.12xlarge': 234, 'm5.24xlarge': 737, 'p2.xlarge': 58, 'p2.8xlarge': 234, 'p2.16xlarge': 234,
        'p3.2xlarge': 58, 'p3.8xlarge': 234, 'p3.16xlarge': 234, 'r3.xlarge': 58, 'r3.2xlarge': 58, 'r3.4xlarge': 234,
        'r3.8xlarge': 234, 'r4.large': 29, 'r4.xlarge': 58, 'r4.2xlarge': 58, 'r4.4xlarge': 234, 'r4.8xlarge': 234,
        'r4.16xlarge': 737, 't2.small': 8, 't2.medium': 17, 't2.large': 35, 't2.xlarge': 44, 't2.2xlarge': 44,
        'x1.16xlarge': 234, 'x1.32xlarge': 234
    }

    def __init__(self, name, cluster_info=None, region=None, subnets=[], tags={}, min_nodes=1, max_nodes=3,
                 role=None, sg_ingresses=[], desired_nodes=1, ami=None, instance_type='m4.large', ssh_public_key=None,
                 keypair=None, kubeconf=None, no_user_data=False):
        self.cluster = cluster_info
        self.subnets = subnets or self.cluster.subnets if self.cluster else []
        self.name = name
        self.tag_name = 'eks.{}.ng.{}'.format(self.cluster.name, self.name)
        self.stack_name = 'eks-{}-ng-{}'.format(self.cluster.name, self.name)
        self.tags = tags
        self.tags.update({'Name': self.tag_name,
                          ControlPlane.CLUSTER_TAG.format(self.cluster.name): 'owned',
                          self.NODEGROUP_TAG: self.name})
        self.region = region
        self.role = role
        self.sg_igresses = sg_ingresses
        self.min = min_nodes
        self.max = max_nodes
        self.desired = desired_nodes
        self.ami = ami
        self.instance = instance_type
        self.ssh_public_key = ssh_public_key or '~/.ssh/id_rsa.pub'
        self.keypair = keypair
        self.keypair_imported = False
        self.ssh_public_key = ssh_public_key or os.path.join(os.path.expanduser("~"), ".ssh", "id_rsa.pub")
        self.tpl = None
        self.use_public_ip = False
        self.kubeconf = kubeconf
        self.resources = None
        self.no_user_data = no_user_data

    def create(self):
        self._validate_creation()
        self._create_cfn_template()
        tags = [{'Key': k, 'Value': v} for (k, v) in viewitems(self.tags)]

        cf = boto3.session.Session().resource('cloudformation')
        reporter = ResourceReporter()
        try:
            stack = cf.create_stack(StackName=self.stack_name, TemplateBody=self.tpl.to_yaml(),
                                    Capabilities=['CAPABILITY_NAMED_IAM'], Tags=tags)

            stack = reporter.report_stack_creation(self.stack_name, listvalues(self.resources), stack.stack_id)
            role = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs}.get(self.RESOURCE_NG_ROLE.name)
        except Exception as e:
            ec2 = boto3.session.Session().resource('ec2')
            if self.keypair_imported:
                r = copy(self.RESOURCE_NG_KEYPAIR)
                r.resource_id = self.keypair
                ec2.KeyPair(self.keypair).delete()
                r.status = Status.deleted
                reporter.succeed(resource=r)

            raise e

        self._update_configmap(os.path.expanduser(self.kubeconf), role)
        return NodeGroupInfo(self.name, self.instance, self.min, self.max, role)

    def query(self, stack=None):
        if not stack:
            cf = boto3.session.Session().resource('cloudformation')
            stack = cf.Stack(self.stack_name)

        rss = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
        asg_name = rss.get(self.RESOURCE_NG_ASG.name).physical_resource_id
        asg_client = boto3.session.Session().client('autoscaling')
        resp = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name], MaxRecords=1)
        asg = resp.get('AutoScalingGroups')[0]
        self.min = asg.get('MinSize')
        self.max = asg.get('MaxSize')
        self.desired = asg.get('DesiredCapacity')

        lc_name = rss.get(self.RESOURCE_NG_ASG_LC.name).physical_resource_id
        resp = asg_client.describe_launch_configurations(LaunchConfigurationNames=[lc_name], MaxRecords=1)
        lc = resp.get('LaunchConfigurations')[0]
        self.instance = lc.get('InstanceType')

        odict = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs}

        return NodeGroupInfo(self.name, self.instance, self.min, self.max, odict.get(self.RESOURCE_NG_ROLE.name))

    @staticmethod
    def _update_configmap(kubeconf, role):
        try:
            load_kubeconf(kubeconf)
            kube = kubernetes.client.CoreV1Api()
            namespace = 'kube-system'
            cm_name = 'aws-auth'
            cm = [item for item in kube.list_namespaced_config_map(namespace).items if item.metadata.name == cm_name]
            if not cm:
                cm = kubernetes.client.V1ConfigMap()
                roles = Environment().from_string(NodeGroup.MAP_ROLE).render(role=role)
                cm.metadata = kubernetes.client.V1ObjectMeta()
                cm.metadata.name = cm_name
                cm.data = {'mapRoles': roles}
                kube.create_namespaced_config_map(namespace=namespace, body=cm)
            else:
                import oyaml as yaml
                cm = cm[0]
                roles = yaml.load(cm.data.get('mapRoles', '[]'))
                node_roles = [r for r in roles if r.get('rolearn') == role]
                if not node_roles:
                    cm.data['mapRoles'] = '\n'.join(
                        [cm.data['mapRoles'], Environment().from_string(NodeGroup.MAP_ROLE).render(role=role)])

                kube.replace_namespaced_config_map('aws-auth', namespace, body=cm)
        except ApiException as e:
            raise EKSCliException('Exception during kubernetes ops: {}'.format(e))

    def _init_resources(self):
        self.resources = OrderedDict([(r.name, r) for r in deepcopy(
            [self.RESOURCE_NG_KEYPAIR, self.RESOURCE_NG_ROLE, self.RESOURCE_NG_PROFILE,
             self.RESOURCE_NG_SG, self.RESOURCE_NG_SG_INGRESS,
             self.RESOURCE_NG_SG_CP_INGRESS,
             self.RESOURCE_CP_SG_INGRESS, self.RESOURCE_CP_EGRESS_TO_NG,
             self.RESOURCE_NG_ASG_LC,
             self.RESOURCE_NG_ASG_LC, self.RESOURCE_NG_ASG])])

    def _validate_creation(self):
        ec2 = boto3.session.Session().resource('ec2')
        if not self.ami:
            self.ami = self.DEFAULT_AMI.get(self.region)
        elif '/' in self.ami:
            owner, name = tuple(self.ami.split('/'))
            images = list(ec2.images.filter(Owners=[owner], Filters=[{'Name': 'name', 'Values': [name]}]))
            if not images:
                raise EKSCliException('image [{}] does not exist.'.format(self.ami))
            self.ami = images[0].id
        else:
            image = ec2.Image(self.ami)
            if not image:
                raise EKSCliException('AMI (id={}) does not exist.'.format(self.ami))

        public_subnet_num = len([s for s in self.subnets if self._is_subnet_public(s)])
        if public_subnet_num != len(self.subnets) and public_subnet_num != 0:
            raise EKSCliException('the subnets [{}] must be either all public or all private.'.format(self.subnets))
        if public_subnet_num == len(self.subnets):
            self.use_public_ip = True

        if not os.path.exists(self.kubeconf):
            raise EKSCliException('kubernetes configuration file ({}) does not exist.'.format(self.kubeconf))

        self._init_resources()

    def _create_cfn_template(self):
        self.tpl = Template()
        self.tpl.add_version('2010-09-09')
        self.tpl.add_description('CFN template to create an EKS node group and affiliated resources.')

        eks_tag = 'kubernetes.io/cluster/{}'.format(self.cluster.name)

        r = self.resources.get(self.RESOURCE_NG_ROLE.name)
        if self.role:
            profile = InstanceProfile(
                self.RESOURCE_NG_PROFILE.name, InstanceProfileName=self.tag_name, Path='/', Roles=[self.role])
            account_id = boto3.session.Session().client('sts').get_caller_identity().get('Account')
            role_arn = 'arn:aws:iam::{}:role/{}'.format(account_id, self.role)
            self.tpl.add_output(
                Output(self.RESOURCE_NG_ROLE.name, Value=role_arn, Description='Node group role'))
            r.status = Status.provided
            r.resource_id = role_arn
        else:
            role = Role(
                self.RESOURCE_NG_ROLE.name, RoleName=self.tag_name,
                AssumeRolePolicyDocument=Policy(Statement=[
                    Statement(Effect=Allow, Action=[AssumeRole],
                              Principal=Principal('Service', ['ec2.amazonaws.com'])), ], ),
                ManagedPolicyArns=['arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy',
                                   'arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy',
                                   'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly']
            )
            self.tpl.add_resource(role)
            profile = InstanceProfile(
                self.RESOURCE_NG_PROFILE.name, InstanceProfileName=self.tag_name, Path='/', Roles=[Ref(role)])
            self.tpl.add_output(
                Output(self.RESOURCE_NG_ROLE.name, Value=GetAtt(role, 'Arn'), Description='Node group role'))

        self.tpl.add_resource(profile)

        if self.sg_igresses:
            sg = SecurityGroup(
                self.RESOURCE_NG_SG.name, VpcId=self.cluster.vpc, Tags=Tags({'Name': self.tag_name, eks_tag: 'owned'}),
                GroupDescription='Security Group applied to the EKS node group',
                SecurityGroupIngress=[SecurityGroupRule(IpProtocol=r.protocol, FromPort=r.from_port, ToPort=r.to_port,
                                                        CidrIp=r.cidr) for r in self.sg_igresses]
            )
        else:
            sg = SecurityGroup(
                self.RESOURCE_NG_SG.name, VpcId=self.cluster.vpc, Tags=Tags({'Name': self.tag_name, eks_tag: 'owned'}),
                GroupDescription='Security Group applied to the EKS node group',
            )
        self.tpl.add_resource(sg)

        self.tpl.add_resource(SecurityGroupIngress(
            self.RESOURCE_NG_SG_INGRESS.name, DependsOn=sg, Description='Allow node to communicate with each other',
            GroupId=Ref(sg), SourceSecurityGroupId=Ref(sg), IpProtocol='-1', FromPort=0, ToPort=65535
        ))

        self.tpl.add_resource(SecurityGroupIngress(
            self.RESOURCE_NG_SG_CP_INGRESS.name, DependsOn=sg,
            Description='Allow kubelet and pods on the nodes to receive communication from the cluster control plane',
            GroupId=Ref(sg), SourceSecurityGroupId=self.cluster.sg, IpProtocol='tcp', FromPort=1025, ToPort=65535
        ))

        self.tpl.add_resource(SecurityGroupEgress(
            self.RESOURCE_CP_EGRESS_TO_NG.name, DependsOn=sg,
            Description='Allow the cluster control plane to communicate with nodes kubelet and pods',
            GroupId=self.cluster.sg, DestinationSecurityGroupId=Ref(sg), IpProtocol='tcp', FromPort=1025, ToPort=65535
        ))

        self.tpl.add_resource(SecurityGroupIngress(
            self.RESOURCE_CP_SG_INGRESS.name, DependsOn=sg,
            Description='Allow pods to communicate with the cluster API Server',
            GroupId=self.cluster.sg, SourceSecurityGroupId=Ref(sg), IpProtocol='tcp', FromPort=443, ToPort=443
        ))

        # keypair
        ec2 = boto3.session.Session().resource('ec2')
        r = self.resources.get(self.RESOURCE_NG_KEYPAIR.name)
        if not self.keypair:
            keyname = 'eks{}'.format(''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(5)))
            with open(self.ssh_public_key, 'rb') as f:
                ec2.import_key_pair(KeyName=keyname, PublicKeyMaterial=f.read())
            self.keypair = keyname
            self.keypair_imported = True
            self.tpl.add_output(Output(self.OUTPUT_KEYNAME, Value=self.keypair, Description='Imported kaypair name'))
            r.status = Status.created
        else:
            r.status = Status.provided

        r.resource_id = self.keypair

        # auto-scaling group and launch configuration
        if self.no_user_data:
            lc = LaunchConfiguration(
                self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
                IamInstanceProfile=Ref(profile),
                ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)])
        else:
            user_data = Base64(
                Join('', [line + '\n' for line in
                          Environment().from_string(self.USER_DATA).render(
                              ci=self.cluster, ng_asg=self.RESOURCE_NG_ASG.name, stack_name=self.stack_name,
                              max_pods=self.MAX_PODS.get(self.instance), region=self.region).split('\n')]))
            lc = LaunchConfiguration(
                self.RESOURCE_NG_ASG_LC.name, AssociatePublicIpAddress=self.use_public_ip,
                IamInstanceProfile=Ref(profile),
                ImageId=self.ami, InstanceType=self.instance, KeyName=self.keypair, SecurityGroups=[Ref(sg)],
                UserData=user_data)

        self.tpl.add_resource(lc)
        self.tpl.add_resource(AutoScalingGroup(
            self.RESOURCE_NG_ASG.name, DesiredCapacity=self.desired, MinSize=self.min, MaxSize=self.max,
            LaunchConfigurationName=Ref(lc), VPCZoneIdentifier=self.subnets,
            Tags=[Tag('Name', self.tag_name, True), Tag(eks_tag, 'owned', True)],
            UpdatePolicy=UpdatePolicy(
                AutoScalingRollingUpdate=AutoScalingRollingUpdate(MinInstancesInService=1, MaxBatchSize=1))))

    @staticmethod
    def _is_subnet_public(subnet):
        ec2 = boto3.session.Session().resource('ec2')
        rts = list(ec2.route_tables.filter(Filters=[{'Name': 'association.subnet-id', 'Values': [subnet]}]))
        for rt in rts:
            for r in rt.routes:
                if r.gateway_id and r.gateway_id.startswith('igw-'):
                    return True

        return False

    def delete(self, stack=None):
        if not stack:
            cf = boto3.session.Session().resource('cloudformation')
            stack = cf.Stack(self.stack_name)

        odict = {o.get('OutputKey'): o.get('OutputValue') for o in stack.outputs or []}
        key_name = odict.get(self.OUTPUT_KEYNAME)

        self._init_resources()
        success = False
        r = self.resources.pop(self.RESOURCE_NG_KEYPAIR.name)
        try:
            success = delete_stack(self.stack_name, list(reversed(listvalues(self.resources))), stack=stack)
        finally:
            if key_name and success:
                reporter = ResourceReporter()
                boto3.session.Session().resource('ec2').KeyPair(key_name).delete()
                r.resource_id = key_name
                r.status = Status.deleted
                reporter.succeed(r)


import logging
import numpy as np
from pylops import LinearOperator
from pylops.utils.backend import get_array_module, \
    get_add_at, to_numpy

logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.WARNING)

def _checkunique(iava):
    _, count = np.unique(iava, axis=1, return_counts=True)
    if np.any(count > 1):
        raise ValueError('Repeated values in iava array')

class Bilinear(LinearOperator):
    r"""Bilinear interpolation operator.

    Apply bilinear interpolation onto fractionary positions ``iava``
    along the first two axes of a n-dimensional array.

    .. note:: The vector ``iava`` should contain unique pais. If the same
       pair is repeated twice an error will be raised.

    Parameters
    ----------
    iava : :obj:`list` or :obj:`numpy.ndarray`
         Array of size :math:`[2 \times n_{ava}]` containing
         pairs of floating indices of locations of available samples
         for interpolation.
    dims : :obj:`list`
        Number of samples for each dimension
    dtype : :obj:`str`, optional
        Type of elements in input array.

    Attributes
    ----------
    shape : :obj:`tuple`
        Operator shape
    explicit : :obj:`bool`
        Operator contains a matrix that can be solved explicitly (``True``) or
        not (``False``)

    Raises
    ------
    ValueError
        If the vector ``iava`` contains repeated values.

    Notes
    -----
    Bilinear interpolation of a subset of :math:`N` values at locations
    ``iava`` from an input n-dimensional vector :math:`\mathbf{x}` of size
    :math:`[m_1 \times m_2 \times ... \times m_{ndim}]` can be expressed as:

    .. math::

        y_{\mathbf{i}} = (1-w^0_{i}) (1-w^1_{i}) x_{l^{l,0}_i, l^{l,1}_i} +
            w^0_{i} (1-w^1_{i}) x_{l^{r,0}_i, l^{l,1}_i} +
            (1-w^0_{i}) w^1_{i} x_{l^{l,0}_i, l^{r,1}_i} +
            w^0_{i} w^1_{i} x_{l^{r,0}_i, l^{r,1}_i}
        \quad \forall i=1,2,...,M

    where :math:`\mathbf{l^{l,0}}=[\lfloor l_1^0 \rfloor,
    \lfloor l_2^0 \rfloor, ..., \lfloor l_N^0 \rfloor]`,
    :math:`\mathbf{l^{l,1}}=[\lfloor l_1^1 \rfloor,
    \lfloor l_2^1 \rfloor, ..., \lfloor l_N^1 \rfloor]`,
    :math:`\mathbf{l^{r,0}}=[\lfloor l_1^0 \rfloor + 1,
    \lfloor l_2^0 \rfloor + 1, ..., \lfloor l_N^0 \rfloor + 1]`,
    :math:`\mathbf{l^{r,1}}=[\lfloor l_1^1 \rfloor + 1,
    \lfloor l_2^1 \rfloor + 1, ..., \lfloor l_N^1 \rfloor + 1]`,
    are vectors containing the indices of the original array at which samples
    are taken, and :math:`\mathbf{w^j}=[l_1^i - \lfloor l_1^i \rfloor,
    l_2^i - \lfloor l_2^i \rfloor, ..., l_N^i - \lfloor l_N^i \rfloor]`
    (:math:`\forall j=0,1`) are the bilinear interpolation weights.

    """
    def __init__(self, iava, dims, dtype='float64'):
        ncp = get_array_module(iava)

        # check non-unique pairs (works only with numpy arrays)
        _checkunique(to_numpy(iava))

        # define dimension of data
        ndims = len(dims)
        self.dims = dims
        self.dimsd = [len(iava[1])] + list(dims[2:])

        # find indices and weights
        self.iava_t = ncp.floor(iava[0]).astype(np.int)
        self.iava_b = self.iava_t + 1
        self.weights_tb = iava[0] - self.iava_t
        self.iava_l = ncp.floor(iava[1]).astype(np.int)
        self.iava_r = self.iava_l + 1
        self.weights_lr = iava[1] - self.iava_l

        # expand dims to weights for nd-arrays
        if ndims > 2:
            for _ in range(ndims - 2):
                self.weights_tb = \
                    ncp.expand_dims(self.weights_tb, axis=-1)
                self.weights_lr = \
                    ncp.expand_dims(self.weights_lr, axis=-1)

        self.shape = (np.prod(np.array(self.dimsd)),
                      np.prod(np.array(self.dims)))
        self.dtype = np.dtype(dtype)
        self.explicit = False

    def _matvec(self, x):
        ncp = get_array_module(x)
        x = ncp.reshape(x, self.dims)
        y = x[self.iava_t, self.iava_l] * (1 - self.weights_tb) * (1 - self.weights_lr) + \
            x[self.iava_t, self.iava_r] * (1 - self.weights_tb) * self.weights_lr + \
            x[self.iava_b, self.iava_l] * self.weights_tb * (1 - self.weights_lr) + \
            x[self.iava_b, self.iava_r] * self.weights_tb * self.weights_lr
        return y.ravel()

    def _rmatvec(self, x):
        ncp = get_array_module(x)
        ncp_add_at = get_add_at(x)
        x = ncp.reshape(x, self.dimsd)
        y = ncp.zeros(self.dims, dtype=self.dtype)
        ncp_add_at(y, [self.iava_t, self.iava_l],
                   x * (1 - self.weights_tb) * (1 - self.weights_lr))
        ncp_add_at(y, [self.iava_t, self.iava_r],
                   x * (1 - self.weights_tb) * self.weights_lr)
        ncp_add_at(y, [self.iava_b, self.iava_l],
                   x * self.weights_tb * (1 - self.weights_lr))
        ncp_add_at(y, [self.iava_b, self.iava_r],
                   x * self.weights_tb * self.weights_lr)
        return y.ravel()


#!/usr/bin/env python3
import sys
import socket
import unittest
from Firewall import Firewall

#from ipaddress import IPv4Address, IPv4Network, IPv4Interface

class FirewallTestCase(unittest.TestCase):

	global firewall
	args = "/Users/kipkoech/Desktop/illumio/Workbook1.csv"

	firewall = Firewall(args)

	def test_parse_range(self):
  		lst1 = [56,93]
  		lst2 = [344, 299999]
  		lst3 = [0, 230000000000]
  		self.assertEqual(firewall.parse_range('56-93'),lst1 )
  		self.assertEqual(firewall.parse_range("344-299999"),lst2 )
  		self.assertEqual(firewall.parse_range("0-230000000000"),lst3 )
  		self.assertEqual(firewall.parse_range("0-230000000000"),lst3 )
  		#print(firewall.parse_range("0-230000000000"))

	def test_checkrange(self):
		self.assertFalse(firewall.checkrange('12-21',10))
		self.assertEqual(firewall.checkrange('0-12030303020202', 2), True)
		self.assertEqual(firewall.checkrange('0-120', 250), False)

	def test_match_ips (self):
		args = "/Users/kipkoech/Desktop/illumio/Workbook1.csv"
		firewall = Firewall(args)
		self.assertTrue(firewall.match_ips('192.168.1.1-192.168.2.5'),1921681119216825)
		print(firewall.match_ips('192.168.1.1-192.168.2.5'))
		self.assertTrue(firewall.match_ips('192.168.1.1-192.168.2.5'),1921681119216825)

	def test_accept(self):
		args = "/Users/kipkoech/Desktop/illumio/Workbook1.csv"
		self.assertFalse(firewall.accept('inbound','uop',80,'192.168.1.2'))
		self.assertFalse(firewall.accept('inbound','tcO',80,'192.168.1.2'))
		self.assertTrue(firewall.accept('inbound','udp',53,'192.168.1.1'))
		self.assertTrue(firewall.accept('inbound','udp',53,'192.168.2.5'),True)
		self.assertTrue(firewall.accept('outbound','udp', 1500,'52.12.48.92'))
if __name__ == '__main__':
    unittest.main()


import warnings
warnings.filterwarnings("ignore")

from selenium import webdriver
import pandas as pd
from selenium.common.exceptions import TimeoutException
import time
import lxml.html as LH
import yaml

def movie_budget_n_metacritic_scrape(titles):
    from datetime import datetime # this import was not working somehow when it was above so brought it here
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--window-size=800x800')
    options.add_argument('--disable-gpu')
    options.add_argument('--no-sandbox')
    options.add_argument('log-level=3')
    # options.add_argument('--proxy-server=45.76.226.206:3128')

    config = yaml.safe_load(open('./../config.yml'))
    data_folder = config['movies_data_folder']

    driver = webdriver.Chrome(config['chromedriver'], chrome_options=options)

    scrape_start_time = datetime.now()
    i = 1
    j = 0

    try:
        df_movie_budget_n_metacritic_already_scraped = pd.read_csv(data_folder+'movie_budget_n_metacritic.csv')
        titles_scraped = list(df_movie_budget_n_metacritic_already_scraped['title_id'])
        del df_movie_budget_n_metacritic_already_scraped
    except:
        titles_scraped = []
    print('Scraping history read...')

    df_main = pd.DataFrame()

    for title_id in titles:
        if titles_scraped.count(title_id) == 0:
            try:
                url = "http://www.imdb.com/title/"+title_id
                driver.get(url)

                try:
                    title_wrapper = driver.find_element_by_class_name('title_wrapper')
                    title_name = title_wrapper.text.split('\n')[0].split(' (')[0].strip()
                    title_year = int(title_wrapper.find_element_by_tag_name('a').text)
                    go_ahead = True
                except:
                    errors = driver.find_elements_by_class_name('error_message')
                    if errors:
                        if errors[0].text.count('URL was not found') == 0:
                            print('Sleeping for 2 minutes...')
                            time.sleep(2*60)
                            try:
                                driver.get(url)
                                title_wrapper = driver.find_element_by_class_name('title_wrapper')
                                title_name = title_wrapper.text.split('\n')[0].split(' (')[0].strip()
                                title_year = int(title_wrapper.find_element_by_tag_name('a').text)
                                go_ahead = True
                            except:
                                go_ahead = False
                        else:
                            go_ahead = False
                    else:
                        if driver.find_element_by_xpath("//*").text == '':
                            print('Sleeping for 2 minutes...')
                            time.sleep(2*60)
                            try:
                                driver.get(url)
                                title_wrapper = driver.find_element_by_class_name('title_wrapper')
                                title_name = title_wrapper.text.split('\n')[0].split(' (')[0].strip()
                                title_year = int(title_wrapper.find_element_by_tag_name('a').text)
                                go_ahead = True
                            except:
                                go_ahead = False
                        else:
                            go_ahead = False

                if go_ahead:
                    subtext = title_wrapper.find_element_by_class_name('subtext')
                    try:
                        release_date = subtext.text.split('|')[-1].strip()
                        run_time = subtext.find_element_by_tag_name('time')
                        if run_time:
                            run_time = run_time.text
                    except:
                        release_date = None
                        run_time = None
                    try:
                        rating_element = driver.find_element_by_class_name('imdbRating')
                        imdb_rating = rating_element.find_element_by_tag_name('strong').text
                        num_votes = rating_element.find_element_by_tag_name('a').text.replace(',', '')
                    except:
                        imdb_rating = None
                        num_votes = None

                    try:
                        summary_text = driver.find_element_by_class_name('summary_text').text
                    except:
                        summary_text = None

                    try:
                        metacritic_score = driver.find_element_by_class_name('titleReviewBarItem').text.split('\n')[0]
                    except:
                        metacritic_score = None

                    try:
                        reviews = driver.find_element_by_css_selector('.titleReviewBarItem.titleReviewbarItemBorder').text
                    except:
                        reviews = None

                    try:
                        awards = driver.find_element_by_css_selector('.article.highlighted').text
                    except:
                        awards = None

                    try:
                        details = driver.find_element_by_id('titleDetails').text
                    except:
                        details = None

                    try:
                        elements = driver.find_elements_by_css_selector('.see-more.inline.canwrap')
                        for element in elements:
                            if element.text.count('Genres:') != 0:
                                genres = [x.strip() for x in element.text.replace('Genres:', '').split('|')]
                    except:
                        genres = None

                    df = pd.DataFrame(
                                      [
                                        {
                                        'title_id':title_id,
                                        'title_name':title_name,
                                        'imdb_rating': imdb_rating,
                                        'num_votes': num_votes,
                                        'release_date': release_date,
                                        'run_time': run_time,
                                        'title_year': title_year,
                                        'genres': genres,
                                        'summary_text':summary_text,
                                        'metacritic_score':metacritic_score,
                                        'reviews':reviews,
                                        'awards':awards,
                                        'details':details
                                        }
                                      ]
                                      )

                    if not df.empty:
                        df_main = pd.concat([df_main,df], axis=0)
                        del df
                else:
                    print('Skipping', title_id, 'as code broke for it.')
                    j += 1
            except TimeoutException as ex:
                titles.append(title_id)
                print('\n')
                print('Skipping '+title_id+' because of Selenium TimeoutException')
                print('\n')
                del driver
                driver = webdriver.Chrome(config['chromedriver'], chrome_options=options)
            if i%25 == 0:
                print('Movies scraped -',(i+j))

                time_since_start = (datetime.now()-scrape_start_time).seconds
                all_time_scraping_speed = (i/time_since_start)*3600
                if time_since_start < 60:
                    time_since_start = str(time_since_start)+' seconds'
                elif time_since_start < 3600:
                    time_since_start = str(time_since_start//60)+ ':'+str(time_since_start%60)+' minutes'
                else:
                    time_since_start = str(time_since_start//3600)+ ':'+str((time_since_start%3600)//60)+' hours'
                print('Time since scraping started - '+time_since_start)
                print('All time scraping speed - '+('%.0f'%(all_time_scraping_speed))+' movies/hour')

                try:
                    time_since_last_checkpoint = (datetime.now()-time_checkpoint).seconds
                except:
                    time_since_last_checkpoint = (datetime.now()-scrape_start_time).seconds
                current_scraping_speed = (25/time_since_last_checkpoint)*3600
                time_remaining = (time_since_last_checkpoint*((len(titles)-i-j)/25))/(3600*24)
                print('Current scraping speed - '+('%.0f'%(current_scraping_speed))+' movies/hour')
                print('Time remaining as per current speed - '+('%.1f'%(time_remaining))+' days')
                print('\n')
                time_checkpoint = datetime.now()

            if i%100 == 0:
                if df_main.shape[0] > 0:
                    try:
                        df_movie_budget_n_metacritic = pd.read_csv(data_folder+'movie_budget_n_metacritic.csv')
                    except:
                        df_movie_budget_n_metacritic = pd.DataFrame()

                    df_movie_budget_n_metacritic = pd.concat([df_movie_budget_n_metacritic,df_main], axis=0)
                    df_movie_budget_n_metacritic.to_csv(data_folder+'movie_budget_n_metacritic.csv', index=False)
                    del df_movie_budget_n_metacritic
                    df_main = pd.DataFrame()
            i += 1
        else:
            j += 1

    if df_main.shape[0] > 0:
        try:
            df_movie_budget_n_metacritic = pd.read_csv(data_folder+'movie_budget_n_metacritic.csv')
        except:
            df_movie_budget_n_metacritic = pd.DataFrame()

        df_movie_budget_n_metacritic = pd.concat([df_movie_budget_n_metacritic,df_main], axis=0)
        df_movie_budget_n_metacritic.to_csv(data_folder+'movie_budget_n_metacritic.csv', index=False)
        del df_movie_budget_n_metacritic
        df_main = pd.DataFrame()

    #################################################################################################################################################################################

                                        #To clean above scraped data

    #################################################################################################################################################################################

    df = pd.read_csv(data_folder+'movie_budget_n_metacritic.csv')

    def country(details):
        try:
            return details.split('Country: ')[1].split('\r')[0].split('\n')[0].split(' | ')
        except:
            return None

    def language(details):
        try:
            return details.split('Language: ')[1].split('\r')[0].split('\n')[0].split(' | ')
        except:
            return None

    def filming_location(details):
        try:
            return details.split('Filming Locations: ')[1].split(' See more')[0].split('\r')[0].split('\n')[0]
        except:
            return None

    def production_house(details):
        try:
            return details.split('Production Co: ')[1].split(' See more')[0].split(',')
        except:
            return None

    def budget(details):
        try:
            return details.split('Budget:')[1].split('\r')[0].split(' (')[0].split('\n')[0]
        except:
            return None

    def opening_weekend_USA(details):
        try:
            return details.split('Opening Weekend USA:')[1].split('\r')[0].split(', ')[0].split('\n')[0]
        except:
            return None

    def gross_USA(details):
        try:
            return details.split('Gross USA:')[1].split('\r')[0].split(', ')[0].split('\n')[0]
        except:
            return None

    def gross_worldwide(details):
        try:
            return details.split('Cumulative Worldwide Gross:')[1].split('\r')[0].split(', ')[0].split('\n')[0]
        except:
            return None

    def reviews_list(reviews):
        try:
            return reviews.split('\n')[1].split(' | ')
        except:
            return None

    def wins_n_nomintations(awards):
        try:
            return awards.split('. See more')[0]
        except:
            return None

    df['country'] = df['details'].apply(lambda x: country(x))
    df['language'] = df['details'].apply(lambda x: language(x))
    df['filming_location'] = df['details'].apply(lambda x: filming_location(x))
    df['production_house'] = df['details'].apply(lambda x: production_house(x))
    df['budget'] = df['details'].apply(lambda x: budget(x))
    df['opening_weekend_USA'] = df['details'].apply(lambda x: opening_weekend_USA(x))
    df['gross_USA'] = df['details'].apply(lambda x: gross_USA(x))
    df['gross_worldwide'] = df['details'].apply(lambda x: gross_worldwide(x))
    del df['details']
    df['reviews_list'] = df['reviews'].apply(lambda x: reviews_list(x))
    del df['reviews']
    df['wins_n_nomintations'] = df['awards'].apply(lambda x: wins_n_nomintations(x))
    del df['awards']

    def no_of_reviews(reviews_list):
        critic = None
        user = None
        try:
            for item in reviews_list:
                if item.split(' ')[1]=='critic':
                    critic = item.split(' ')[0]
                else:
                    user = item.split(' ')[0]
        except:
            pass
        return critic, user

    df['critic_review'], df['user_review'] = zip(*df['reviews_list'].apply(no_of_reviews))
    del df['reviews_list']

    import re
    def total_win_nominations(wins_n_nomintations):
        gp = re.search('^[Nominated for\s]+([\d]+)', str(wins_n_nomintations), re.I)
        try:
            add_nominations = int(gp.group(1))
        except:
            add_nominations = 0

        gp = re.search('^[Won\s]+([\d]+)', str(wins_n_nomintations), re.I)
        try:
            add_wins = int(gp.group(1))
        except:
            add_wins = 0

        gp = re.search('([\d]+)(?= win)', str(wins_n_nomintations), re.I)
        try:
            total_wins = int(gp.group(1))
        except:
            total_wins = 0

        gp = re.search('([\d]+)(?= nomination)', str(wins_n_nomintations), re.I)
        try:
            total_nominations = int(gp.group(1))
        except:
            total_nominations = 0

        total_wins = total_wins+add_wins
        total_wins = total_wins if total_wins else None
        total_nominations = total_nominations+add_nominations
        total_nominations = total_nominations if total_nominations else None

        return total_wins, total_nominations

    df['award_wins'], df['award_nominations'] = zip(*df['wins_n_nomintations'].apply(total_win_nominations))
    del df['wins_n_nomintations']

    def metacritic(score):
        try:
            return int(score)
        except:
            return None
    df['metacritic_score'] = df['metacritic_score'].apply(lambda x: metacritic(x))
    del df['summary_text']

    df.rename(columns={'title_year':'release_year'}, inplace=True)

    from currency_converter import CurrencyConverter
    from datetime import datetime
    c = CurrencyConverter()
    df_curr_xc = pd.read_csv('./../resources/currency_exchange.csv')
    df_inflation = pd.read_csv('./../resources/usd_inflation.csv')
    curr_mapping = {
        '£':'GBP',
        '€':'EUR',
        '$':'USD'
    }
    def normalize_budget(budget, year):
        last_date = None
        currency = None
        try:
            year = int(year)
        except:
            year = None
        if budget and year:
            currency = re.search('^([\D]+)', budget).group(1).strip()
            budget = float(budget.replace(currency,'').replace(',','').replace(' ','').strip())
            currency = curr_mapping.get(currency, currency)
            try:
                if currency != 'USD':
                    budget = c.convert(budget, currency, new_currency='USD', date=datetime.strptime(str(year)+'/1/1', '%Y/%m/%d'))
                price_old = df_inflation['value'][df_inflation['year']==year].asobject[0]
                price_new = df_inflation['value'][df_inflation['year']==2018].asobject[0]
                budget = budget*(price_new/price_old)
                currency = 'USD'
                year = 2018
            except:
                try:
                    if currency != 'USD':
                        rate = df_curr_xc['rate'][(df_curr_xc['currency']==currency) & (df_curr_xc['year']==year)].asobject[0]
                        budget = rate*budget
                    price_old = df_inflation['value'][df_inflation['year']==year].asobject[0]
                    price_new = df_inflation['value'][df_inflation['year']==2018].asobject[0]
                    budget = budget*(price_new/price_old)
                    currency = 'USD'
                    year = 2018
                except:
                    pass
        return {
            'currency': currency,
            'amount': budget,
            'year': year
        }

    df['budget'] = df.apply(lambda row: normalize_budget(row['budget'], row['release_year']), axis=1)
    df['opening_weekend_USA'] = df.apply(lambda row: normalize_budget(row['opening_weekend_USA'], row['release_year']), axis=1)
    df['gross_USA'] = df.apply(lambda row: normalize_budget(row['gross_USA'], row['release_year']), axis=1)
    df['gross_worldwide'] = df.apply(lambda row: normalize_budget(row['gross_worldwide'], row['release_year']), axis=1)

    df.to_csv(data_folder+'cleaned_movie_budget_n_metacritic.csv', index=False)

    return True


from flask import Flask, request, jsonify
from flasgger import Swagger

app = Flask(__name__)
Swagger(app)

@app.route('/')
def home():
    """
    Home endpoint
    ---
    responses:
      200:
        description: Welcome message
    """
    return f'Hello there from Bonnie Flask API Endpoint Server'

@app.route('/hello', methods=['GET'])
def hello_get():
    """
    Hello endpoint with GET method
    ---
    parameters:
      - name: name
        in: query
        type: string
        default: World
        description: The name parameter
      - name: middlename
        in: query
        type: string
        default: no middle name provided
        description: The middle name parameter

    responses:
      200:
        description: Hello message
    """
    name = request.args.get('name', 'World')
    middlename = request.args.get('middlename', 'no middle name provided')
    nameCapital = name.upper()
    middlenameCapital = middlename.upper()
    return f'Hello {nameCapital} {middlenameCapital}!'

@app.route('/hello', methods=['POST'])
def hello_post():
    """
    Hello endpoint with POST method
    ---
    parameters:
      - name: body
        in: body
        required: true
        schema:
          type: object
          properties:
            name:
              type: string
              default: World
              description: The name parameter

    responses:
      200:
        description: Hello message
    """
    data = request.get_json()
    if data is None:
        return jsonify({'error': 'Invalid JSON'}), 400

    name = data.get('name', 'World')
    return jsonify({'message': f'Hello {name}!'})

if __name__ == '__main__':
    app.run(debug=True)


from django.test import TestCase
from ..models import Group, Post, User

class PostModelTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='test_user')
        cls.group = Group.objects.create(
            title='test_group',
            slug='test_slug',
            description='test_description',
        )
        cls.post = Post.objects.create(
            author=cls.user,
            text="""Lorem Ipsum is simply dummy text of the printing and
            typesetting industry.
            Lorem Ipsum has been the industry's standard dummy""",
            group=cls.group,
        )

    def test_model_have_correct_str_in_group(self):
        self.assertEqual(self.group.title, str(self.group))

    def test_model_have_correct_str_in_post(self):
        self.assertEqual(self.post.text[:15], str(self.post))


import os.path
from pathlib import Path
import time
import shutil
import csv
import math

def move_file(src, dest):
    """ Given a file 'src', move that file to the correct file specified by 'dest' """
    # exit with error code 4 if this fails...
    img_moved = False
    while not img_moved:
        try:
            shutil.move(src, dest)
        except OSError as e:
            if e.errno == 13:
                # print('File move error caught with OSError.errno 13') #  for debugging
                # This sleep could be 0.2 secs but I don't want to use this high a sleep time.
                # Could be an issue if the user makes the camera rotation time too low
                time.sleep(0.1)
                continue
            # raise the exception if it is not this particular error
            print('ERROR:')
            print('The image file mover process has encountered some unknown error. Please restart the script.')
            print('See the python error description here:')
            print(e)
            exit(4)
        # explicitly raise any other exception type
        except Exception as weirdE:
            print('ERROR:')
            print('The image file mover process has encountered some unknown error. Please restart the script.')
            print('See the python error description here:')
            print(weirdE)
            exit(4)
        img_moved = True

def image_file_mover(check_delay, config_params, cam_pos_list):
    """ Parameters:
            check_delay: how long to wait between checking for image file existence
            config_params: a dictionary of relevant configuration parameters used in this run of the program
            cam_pos_list: the list of generated camera positions
        Description:
        This function will check a pre-determined directory for pre-determined file name and will continue to check,
    sleeping for <check_delay> seconds between each check. If the file exists, it will move it to a separate
    pre-determined directory. check_delay should be 1/2 the time it takes to rotate the camera to the next position.
        Future work could be to set the process affinity here to a different core, but isn't important right now.
    """
    img_ctr = 1
    blank_ctr = 1
    num_cam_positions = int(1 + math.floor(359/config_params['cam_rota_angle_step']))
    print("Number of camera positions: %d" % num_cam_positions)
    num_vehicle_rotations = int(1 + math.floor(359/config_params['vehicle_rota_angle_step']))
    print("Number of vehicle rotations: %d" % num_vehicle_rotations)
    num_fog_settings = int(1 + math.floor(0.8/config_params['fog_incr']))
    print("Number of fog settings: %d" % num_fog_settings)
    num_time_settings = int(1 + math.floor(13.5/config_params['time_incr']))  # from 6 to 19.5
    print("Number of time settings: %d" % num_time_settings)
    num_vehicles = len(config_params['vehicle_names'])
    print("Number of vehicles: %d" % num_vehicles)
    num_map_positions = len(config_params['map_positions'])
    print("Number of map positions: %d" % num_map_positions)
    num_images = num_map_positions * num_vehicles * num_cam_positions * num_vehicle_rotations * num_fog_settings * num_time_settings
    images_per_vehicle = num_images/num_vehicles
    images_per_vehicle_at_pos = images_per_vehicle/num_map_positions
    images_per_vehicle_per_tod = images_per_vehicle_at_pos/num_time_settings
    images_per_vehicle_per_fog = images_per_vehicle_per_tod/num_fog_settings

    # vehicle is the top level loop construct, then map positions, then time of day, then fog, then vehicle
    # orientation, then camera rotations.

    # relative counters for tracking occurrences
    map_pos_idx = 0
    vehicle_idx = 0
    fog_ct = 0
    tod_ct = 0
    veh_orient_ct = 0  # number of vehicle orientations seen
    cam_pos_ct = 0

    screenshot_path = Path(os.path.expanduser('~\\Documents\\Arma 3\\Screenshots'))
    dest_dir = Path(os.path.expanduser('~\\Documents\\SyntheticDataGen'))
    if not Path.exists(dest_dir):
        os.makedirs(str(dest_dir), 0o777)  # create the directory if it doesnt exist
    # if not screenshot_path.exists(): # I will fix this later
        # need to do some error stuff here
    # open annotation csv file and write the header data. THIS WILL BLOW AWAY THE FILE CONTENTS
    annotation_file_path = Path(str(dest_dir) + '\\AnnotationData.csv')
    annotation_csv = open(annotation_file_path, 'w+', newline='')
    annotation_writer = csv.writer(annotation_csv)
    annotation_writer.writerow(['filename','vehiclename','target_x','target_y','target_z','camera_x','camera_y','camera_z',
                               'foglevel','time','TR','BR','TL','BL'])
    annotation_csv.close()
    # re-open file for append to write the data. I will leave this file open for time concerns while moving files
    annotation_csv = open(annotation_file_path, 'a', newline='')
    annotation_writer = csv.writer(annotation_csv)
    print('The camera position list is: %s' % cam_pos_list)
    # while img_ctr <= num_images and blank_ctr <= num_images:
    while img_ctr <= num_images:
        screenshot = Path(str(screenshot_path) + '\\arma3screenshot.png')
        blank_screenshot = Path(str(screenshot_path) + '\\arma3blank.png')
        # check for data screenshot
        if screenshot.exists() and not screenshot.is_dir():
            # exit with error code 4 if this fails...
            dest = Path(os.path.expanduser('~\\Documents\\SyntheticDataGen\\' + str(img_ctr) + '.png'))
            move_file(screenshot, dest)

            # append this file's metadata to the annotation csv file here
            filename = '%s.png' % img_ctr
            time_frac, time_hr = math.modf(6 + tod_ct * config_params['time_incr'])
            time_min = int(round(time_frac * 60))
            timestr = '%d:%d' % (time_hr, time_min)
            data = [filename,
                    config_params['vehicle_names'][vehicle_idx],
                    config_params['map_positions'][map_pos_idx][0],
                    config_params['map_positions'][map_pos_idx][1],
                    config_params['map_positions'][map_pos_idx][2],
                    cam_pos_list[cam_pos_ct][0],
                    cam_pos_list[cam_pos_ct][1],
                    cam_pos_list[cam_pos_ct][2],
                    fog_ct * config_params['fog_incr'],
                    timestr]
            annotation_writer.writerow(data)
            cam_pos_ct = cam_pos_ct + 1
            # calculate where in each environmental effect the next image will be
            if cam_pos_ct == num_cam_positions:
                cam_pos_ct = 0
                veh_orient_ct = veh_orient_ct + 1
                if veh_orient_ct == num_vehicle_rotations:
                    veh_orient_ct = 0
                    fog_ct = fog_ct + 1
                    if fog_ct == num_fog_settings:
                        fog_ct = 0
                        tod_ct = tod_ct + 1
                        if tod_ct == num_time_settings:
                            tod_ct = 0
                            map_pos_idx = map_pos_idx + 1
                            if map_pos_idx == num_map_positions:
                                map_pos_idx = 0
                                vehicle_idx = vehicle_idx + 1

            img_ctr += 1  # increment the image counter after a successful move
            # I have never seen it print these
            if vehicle_idx == num_vehicles and img_ctr <= num_images:
                print("There appears to be an error counting images...")
                print("Current number of vehicles evaluated: %d" % vehicle_idx)
                print("Current number of images seen: %d / %d" % (img_ctr, num_images))

        # if data screenshot is not present, check for img comparison screenshot
        elif blank_screenshot.exists() and not blank_screenshot.is_dir():
            dest = Path(os.path.expanduser('~\\Documents\\SyntheticDataGen\\' + str(blank_ctr) + '_blank.png'))
            move_file(blank_screenshot, dest)
            blank_ctr += 1  # increment the image counter after a successful move

        time.sleep(check_delay)  # sleep for check_delay seconds so we don't hammer the cpu unnecessarily

    annotation_csv.close()


import astropy.io.fits
import numpy as np

# list of filenames to stack
images = ['12jan2019/co2_deep-H_262-419.fits','15mar2019/co2_deepH_mar2019_123-206.fits']
# list of weights for each images
weights = [5, 3]
# put a 1 if the images should be inverted
inverts = [0, 1]

#output file
output='combine.fits'
total = 0

# loop through each image
for invert, img, w in zip(inverts, images, weights):

    # read in and normalize each image
    data = astropy.io.fits.getdata(img)
    data = data / np.abs(np.median(data))

    # check if it is the first image read in
    if np.sum(total) == 0:
        total = w * data / np.sum(weights)

    else:
        # if you want it to be inverted then do that
        if invert:
            total -= w * data / np.sum(weights)
        else:
            total += w * data / np.sum(weights)

# write file
astropy.io.fits.writeto(output, total, overwrite=True)