# Copyright (c) 2016-2018, University of Idaho
# All rights reserved.
#
# Roger Lew (rogerlew@gmail.com)
#
# The project described was supported by NSF award number IIA-1301792
# from the NSF Idaho EPSCoR Program and by the National Science Foundation.

from typing import Tuple, List, Dict, Union
import os
from os.path import join as _join
from os.path import exists as _exists
import math
import json
from collections import Counter
from math import pi, atan2
import warnings

import numpy as np
from scipy.stats import circmean

from wepppy.all_your_base import (
    isfloat,
    read_arc,
    wgs84_proj4,
    centroid_px,
    GeoTransformer
)
from .wepp_top_translator import WeppTopTranslator

_thisdir = os.path.dirname(__file__)
_template_dir = _join(_thisdir, "templates")

def ischannel(topaz_id: int) -> bool:
    return str(topaz_id).endswith('4')

def garbrecht_length(distances: List[List[float]]) -> float:
    """
    calculates the length of a subcatchment from the flowpaths
    contained within the subcatchment. The length is weighted_flowpaths
    by the flowpaths length relative to its area.

    distances should be an array of distances between cells along the
    flowpath (not cumulative distance)

    eq. 3.4 in Thomas Cochrane's Dissertation
    # """
    x = np.array([np.sum(d) for d in distances])
    a = np.array([len(d) for d in distances], dtype=np.float64)
    return float(np.sum(x * a) / np.sum(a))

def cummnorm_distance(distance: List[float]) -> np.array:
    """
    builds and returns cumulative normalized distance array from an array
    of cell-to-cell distances
    """
    assert len(distance) > 0

    if len(distance) == 1:
        assert distance[0] > 0.0
        return np.array([0, 1])

    distance_p = np.cumsum(np.array(distance, np.float64))
    distance_p -= distance_p[0]
    distance_p /= distance_p[-1]
    return distance_p

def representative_normalized_elevations(x: List[float], dy: List[float]) -> List[float]:
    """
    x should be a normed distance array between 0 and 1
    dy is an array of slopes

    returns normalized elevations (relative to the length of x)
    """
    assert len(x) == len(dy), (x, dy)
    assert x[0] == 0.0
    assert x[-1] == 1.0

    # calculate the positions, assume top of hillslope is 0 y
    y = [0.0]
    for i in range(len(dy) - 1):
        step = x[i+1] - x[i]
        y.append(y[-1] - step * dy[i])

    return y

def read_geojson(fname):
    data = json.loads(open(fname).read())

    d = {}
    for feature in data['features']:
        top = feature['properties']['TopazID']
        d[top] = np.array(feature['geometry']['coordinates'][0])

    return d

def interpolate_slp(distances, slopes, max_points):
    _s = np.array(slopes)
    _d = np.array(distances)

    if _s.shape == (1,) and _d.shape == (2,):  # slope is a single cell
        _s = np.array([slopes[0], slopes[0]])

    assert _s.shape == _d.shape, str([_s.shape, _d.shape])

    for i in range(len(_d)-1):
        assert _d[i] < _d[i+1], distances

    assert _d[0] == 0.0
    assert _d[-1] == 1.0

    npts = len(_d)

    # interpolate if there are too many slope points
    if npts > max_points:
        _d2 = np.linspace(0.0, 1.0, max_points)
        _s2 = np.interp(_d2, _d, _s)
        _d, _s = _d2, _s2

    return _d, _s

def write_slp(aspect, width, cellsize, length, slope, distance_p, fp, version=97.3, max_points=19):
    """
    writes a slope file in the 97.3 format for WEPP
    """
    assert isfloat(aspect)
    assert isfloat(width)
    assert isfloat(length)
    assert isfloat(cellsize)

    _d, _s = distance_p, slope

    nofes = 1
    npts = len(_d)
    if npts > max_points:
        _d, _s = interpolate_slp(distance_p, slope, max_points)
        npts = len(_d)

    if version == 97.3:
        _slp = '97.3\n{nofes}\n{aspect} {width}\n{npts} {length}\n{defs} '
        defs = ' '.join(['%0.4f, %0.5f' % ds for ds in zip(_d, _s)])

        fp.write(_slp.format(nofes=nofes, aspect=aspect, width=width,
                             npts=npts, length=length, defs=defs))

    else:
        _slp = '{aspect} {cellsize}\n{npts} {length}\n{defs} \n'
        defs = ' '.join(['%0.4f %0.5f' % ds for ds in zip(_d, _s)])

        fp.write(_slp.format(nofes=nofes, aspect=aspect, cellsize=float(cellsize),
                             npts=npts, length=length, defs=defs))

def _identify_subflows(flowpaths: List[np.array]) -> List[List[int]]:
    """
    given an ndarray of flowpaths flowpath (n, 2) for a subcatchment
    identify which flowpaths travel over the same path down the
    hillslope. These are grouped into subflows of flowpath indices
    """
    # if there is only 1 flowpath then there are no subflows
    if len(flowpaths) == 1:
        return [[0]]

    # create a dictionary of px coord tuples so we can point back to the
    # index of the flowpaths list. This is aligned
    # with slopes and distance.
    fps_d = {}
    for k, fp in enumerate(flowpaths):
        fps_d[k] = list(set((fp[j, 0], fp[j, 1]) for j in range(fp.shape[0])))

    # here we create a list of tuples containing
    # the flow indices and the length of the flows
    # we want to iterate through the flows in
    # descending flow length
    lns = [(k, len(v)) for k, v in fps_d.items()]

    # build an empty list to populate subflows
    subflows = []

    # iterate over the flowpaths. If a subflow is identified as
    # a subflow then add it to the subflows index list
    # otherwise create a new subflow list
    for i, n in sorted(lns, key=lambda y: y[1], reverse=True):
        # unpack the set of pixel coords for the subflow
        fp0 = fps_d[i]

        # this gets set to 1 if fp0 is identified as a subflow
        issub = 0

        for j, fp1_indx_arr in enumerate(subflows):
            # because we are iterating in descending order the
            # first index of each subflow index list will
            # be the longest
            fp1 = fps_d[fp1_indx_arr[0]]

            # is fp0 a subset of fp1
            if fp0 <= fp1:
                subflows[j].append(i)
                issub = 1
                break

        # fp0 is not a subset so create a new subflow list
        if not issub:
            subflows.append([i])

    return subflows

def _weighted_slope_average(flowpaths, slopes, distances, max_points=19):
    """
    calculates weighted slopes based on the flowpaths contained on the hillslope
    """
    # determine longest flowpath
    lengths = [float(np.sum(d)) for d in distances]
    i = int(np.argmax(lengths))
    longest = float(lengths[i])

    # determine number of points to define slope
    num_points = len(distances[i])
    if num_points > max_points:
        num_points = max_points

    if num_points == 1:
        slope = float(slopes[i])
        return [slope, slope], [0.0, 1.0]

    # for each flowpath determine the distance from channel
    # this requires reversing the elements in the distance
    # array and calculating the cumulative sum
    rev_cum_distances = [np.cumsum(d[::-1]) for d in distances]

    # if we did this right, then the distance should equally the longest
    # length
    assert round(longest, 5) == round(rev_cum_distances[i][-1], 5)

    # determine weights for each flowpath
    areas = [f.shape[0] for f in flowpaths]
    kps = np.array([L * a for L, a in zip(lengths, areas)])

    # build an array with equally spaced points to interpolate on
    distance_p = np.linspace(0, longest, num_points)

    # this will hold the weighted slope estimates
    eps = []

    # we will weight the slope at each distance away from the channel
    for d_p in distance_p:
        num = 0  # to hold numerator value
        kpsum = 0  # to hold k_p sum

        for slp, rcd, kp in zip(slopes, rev_cum_distances, kps):
            # reverse slopes
            slp = slp[::-1]

            # we only want to interpolate where the slope is defined
            if d_p - 1e-6 > rcd[-1]:
                continue

            slp_p = np.interp(d_p, rcd, slp)
            num += slp_p * kp
            kpsum += kp

        # store the weighted slope estimate
        eps.append(num / kpsum)

    # normalize distance_p array
    distance_p /= longest

    # reverse weighted slopes and return
    w_slopes = np.array(eps[::-1])

    return w_slopes.flatten().tolist(), distance_p.tolist()

def compute_direction(head: List, tail: List) -> float:
    a = atan2(tail[1] - head[1],
              head[0] - tail[0]) * (180.0 / pi) - 180.0

    if a < 0:
        return a + 360.0

    return a

slope_template = """\
{aspect} {profile_width}
{num_points} {length}
{profile}"""

colordict = {
  0:     "#787878",
  "n":  ["#9afb0c", "#9ddd5e", "#9fc085"],
  "ne": ["#00ad43", "#3dab71", "#72a890"],
  "e":  ["#0068c0", "#5078b6", "#7c8ead"],
  "se": ["#6c00a3", "#77479d", "#8c75a0"],
  "s":  ["#ca009c", "#c04d9c", "#b47ba1"],
  "sw": ["#ff5568", "#e76f7a", "#cb8b8f"],
  "w":  ["#ffab47", "#e2a66c", "#c5a58a"],
  "nw": ["#f4fa00", "#d6db5e", "#bdbf89"]
}

c_slope_breaks = [0.05, 0.15, 0.30]

def slp_asp_color(slope: float, aspect: float) -> str:
    aspect %= 360.0

    i = 0
    for j, brk in enumerate(c_slope_breaks):
        if slope > brk:
            i = j + 1

    if i == 0:
        return colordict[0]

    cat_asp = "n"
    if 22.5 < aspect < 67.5:
        cat_asp = "ne"
    elif aspect < 112.5:
        cat_asp = "e"
    elif aspect < 157.5:
        cat_asp = "se"
    elif aspect < 202.5:
        cat_asp = "s"
    elif aspect < 247.5:
        cat_asp = "sw"
    elif aspect < 292.5:
        cat_asp = "w"
    elif aspect < 337.5:
        cat_asp = "nw"

    return colordict[cat_asp][i-1]

class ChannelRoutingError(Exception):
    """
    THE NETWORK STRUCTURE DELINEATED BY TOPAZ CONTAINS MORE THAN 3 CHANNELS
    AS INPUT TO A SINGLE CHANNEL. WEPP CAN ONLY HANDLE 3 OR LESS CHANNELS
    DRAINING INTO A SINGLE CHANNEL.

    THE CHANNELS SHOULD BE REDELINEATED AFTER ADJUSTING THE MINIMUM CHANNEL
    LENGTH (MCL) AND/OR CRITICAL SOURCE AREA (CSA)
    """

    __name__ = 'ChannelRoutingError'

    def __init__(self):
        pass

class CentroidSummary(object):
    def __init__(self, **kwds):
        self.px = tuple(int(v) for v in kwds['px'])
        self.lnglat = tuple(float(v) for v in kwds['lnglat'])

class SummaryBase(object):
    def __init__(self, **kwds):
        topaz_id = kwds['topaz_id']
        if topaz_id is not None:
            topaz_id = int(topaz_id)
        self.topaz_id = topaz_id

        wepp_id = kwds['wepp_id']
        if wepp_id is not None:
            wepp_id = int(wepp_id)
        self.wepp_id = wepp_id

        self.length = float(kwds['length'])
        self.width = float(kwds['width'])
        self.area = float(kwds['area'])
        self.aspect = float(kwds['aspect'])
        self.direction = float(kwds['direction'])
        self.slope_scalar = float(kwds['slope_scalar'])
        self.color = str(kwds['color'])
        self.centroid = kwds['centroid']
        self.distance_p = tuple(kwds['distance_p'])

        self._max_points = 19

        self.w_slopes = None
        self.slopes = None

    @property
    def num_points(self) -> int:
        npts = len(self.distance_p)
        max_points = self.max_points
        return (npts, max_points)[npts > max_points]

    @property
    def max_points(self) -> int:
        return self._max_points

    @property
    def profile(self) -> str:
        max_points = self.max_points
        distance_p = self.distance_p

        try:
            slopes = self.w_slopes
            if slopes is None:
                slopes = self.slopes
        except AttributeError:
            slopes = self.slopes

        _d, _s = distance_p, slopes

        npts = len(_d)
        if npts > max_points:
            _d, _s = interpolate_slp(_d, _s, max_points)

        return ' '.join(["%0.6f, %0.3f" % (d, s) for d, s in zip(_d, _s)])

    def as_dict(self):
        d = dict(
            fname=self.fname,
            topaz_id=self.topaz_id,
            wepp_id=self.wepp_id,
            length=self.length,
            width=self.width,
            area=self.area,
            aspect=self.aspect,
            direction=self.direction,
            slope_scalar=self.slope_scalar,
            color=self.color,
            centroid=self.centroid.lnglat
        )

        try:
            d['order'] = self.order
            d['channel_type'] = self.channel_type
            d['cell_width'] = self.cell_width
        except:
            pass

        try:
            d['slopes'] = self.slopes
            d['coords'] = self.coords
        except:
            pass

        return d

class HillSummary(SummaryBase):
    def __init__(self, **kwds):
        super(HillSummary, self).__init__(**kwds)
        self.w_slopes = tuple(kwds['w_slopes'])
        self.pourpoint = tuple(kwds['pourpoint'])

    @property
    def fname(self) -> str:
        return 'hill_%i.slp' % self.topaz_id

    @property
    def pourpoint_coord(self):
        return str(tuple([int(v) for v in self.pourpoint])).replace(' ', '')

class ChannelSummary(SummaryBase):
    def __init__(self, **kwds):
        super(ChannelSummary, self).__init__(**kwds)
        self.slopes = tuple(kwds['slopes'])
        self._order = kwds.get('order', None)
        self.isoutlet = kwds['isoutlet']
        self.head = kwds['head']
        self.tail = kwds['tail']
        self.chn_enum = int(kwds['chn_enum'])
        self._chn_type = kwds['chn_type']

    @property
    def head_coord(self):
        return str(tuple([int(v) for v in self.head])).replace(' ', '')

    @property
    def tail_coord(self):
        return str(tuple([int(v) for v in self.tail])).replace(' ', '')

    @property
    def fname(self) -> str:
        return 'chn_%i.slp' % self.topaz_id

    @property
    def order(self) -> int:
        return self._order

    @order.setter
    def order(self, value):
        self._order = int(value)

    @property
    def channel_type(self) -> str:
        return getattr(self, '_chn_type', 'Default')

        # return 'Default'
        # return ('OnEarth 1', 'OnGravel 1', 'OnRock 2')[int([None, 0, 0, 1, 1, 1, 2, 2][self.order])]

    @channel_type.setter
    def channel_type(self, value):
        self._chn_type = value

    @property
    def cell_width(self) -> int:
        return [None, 1, 2, 2, 3, 3, 3, 4][self.order]

class FlowpathSummary(SummaryBase):
    def __init__(self, **kwds):
        super(FlowpathSummary, self).__init__(**kwds)
        self.slopes = tuple(kwds['slopes'])
        self.coords = kwds['coords']

    @property
    def fname(self) -> str:
        return 'flow_%i.slp' % self.topaz_id

class WatershedAbstraction:
    def __init__(self, wd, wat_dir):
        if not _exists(wd):
            raise Exception('Specified working directory does not exist')

        if not _exists(wat_dir):
            raise Exception('Specified wat_dir does not exist')

        self.wd = wd
        self.wat_dir = wat_dir

        # lookup table for the flowpath directions specified in flovec
        #
        # 1     2     3
        #   \   |   /
        #     \ | /
        # 4 <-- o --> 6
        #     / | \
        #   /   |   \
        # 7     8     9
        #
        # the origin for the maps is the top right corner
        # we read in the maps with gdal and transpose so the
        # data is in row major order. This is less consistent
        # with maps but more consistent with numpy
        sqrt2 = 2.0**0.5
        self.paths = {1: ([-1, -1], sqrt2),
                      2: ([0,  -1], 1),
                      3: ([1,  -1], sqrt2),
                      4: ([-1,  0], 1),
                      6: ([0,   1], 1),
                      7: ([-1,  1], sqrt2),
                      8: ([0,   1], 1),
                      9: ([1,   1], 1)}

        # initialize datastructure containing json representation
        # of the abstracted watershed
        self.watershed = dict(hillslopes={}, channels={}, flowpaths={})
        self.hillslope_n = 0
        self.channel_n = 0
        self.impoundment_n = 0
        self._structure = None
        self._linkdata = None
        self._network = None

        bound_fn = _join(wd, 'BOUND.ARC')
        flopat_fn = _join(wd, 'FLOPAT.ARC')
        flovec_fn = _join(wd, 'FLOVEC.ARC')
        fvslop_fn = _join(wd, 'FVSLOP.ARC')
        subwta_fn = _join(wd, 'SUBWTA.ARC')
        relief_fn = _join(wd, 'RELIEF.ARC')
        taspec_fn = _join(wd, 'TASPEC.ARC')
        dnmcnt_inp = _join(wd, 'DNMCNT.INP')

        if not _exists(bound_fn) or \
           not _exists(flopat_fn) or \
           not _exists(flovec_fn) or \
           not _exists(fvslop_fn) or \
           not _exists(subwta_fn) or \
           not _exists(relief_fn) or \
           not _exists(taspec_fn):
            raise Exception('Missing required input file')

        bound, _transform, _proj = read_arc(bound_fn, dtype=np.uint8)
        flopat, _transform, _proj = read_arc(flopat_fn, dtype=np.uint8)
        flovec, _transform, _proj = read_arc(flovec_fn, dtype=np.uint8)
        fvslop, _transform, _proj = read_arc(fvslop_fn, dtype=np.float64)
        subwta, _transform, _proj = read_arc(subwta_fn, dtype=np.int64)
        relief, _transform, _proj = read_arc(relief_fn, dtype=np.float64)
        taspec, _transform, _proj = read_arc(taspec_fn, dtype=np.float64)

        # make sure the datasets are aligned
        assert flopat.shape == flovec.shape
        assert flopat.shape == fvslop.shape
        assert flopat.shape == subwta.shape
        assert flopat.shape == relief.shape
        assert flopat.shape == taspec.shape

        # clamp slope values at 0.001 to prevent internal
        # errors in wepp
        fvslop[np.where(fvslop < 0.001)] = 0.001

        # pull cellsize from transform
        self.transform = _transform
        self.cellsize = _transform[1]
        self.cellsize2 = self.cellsize * self.cellsize
        self.totalarea = float(np.sum(bound)) * self.cellsize2

        # store in class
        self.flopat = flopat
        self.flovec = flovec
        self.fvslop = fvslop
        self.subwta = subwta
        self.relief = relief
        self.taspec = taspec

        # find outlet
        outlet = self._read_outlet(dnmcnt_inp)
        outlet_top_id = subwta[outlet[0], outlet[1]]
        assert str(outlet_top_id).endswith('4'), outlet_top_id

        self.watershed['outlet'] = outlet
        self.outlet_top_id = outlet_top_id
        self.watershed['srs'] = dict(transform=_transform,
                                     projection=_proj)

        self.utmProj = _proj
        self.proj2wgs_transformer = GeoTransformer(src_proj4=self.utmProj, dst_proj4=wgs84_proj4)

        # find the centroid_px for the watershed
        indx, indy = np.where(bound == 1)
        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        self._centroid = CentroidSummary(px=_centroid_px,
                                         lnglat=centroid_lnglat)

        top_ids = list(set(subwta.flatten().tolist()))
        top_ids.remove(0)
        top_sub_ids = [v for v in top_ids if str(v)[-1] in '123']
        top_chn_ids = [v for v in top_ids if str(v).endswith('4')]

        self.translator = WeppTopTranslator(top_sub_ids, top_chn_ids)

    @property
    def structure(self) -> List[List[int]]:
        return self._structure

    @property
    def linkdata(self) -> Dict[int, Dict[str, List[Union[int, float]]]]:
        return self._linkdata

    @property
    def network(self) -> Dict[int, List[int]]:
        return self._network

    @property
    def centroid(self) -> CentroidSummary:
        return self._centroid

    def abstract(self, wepp_chn_type='Default', verbose=False, warn=False):
        self.abstract_channels(wepp_chn_type=wepp_chn_type, verbose=verbose)
        self.abstract_subcatchments(verbose=verbose, warn=warn)
        self.abstract_structure(verbose=verbose)

    def write_slps(self, channels=1, subcatchments=1, flowpaths=1, cell_width=None):
        """
        Writes slope files to the specified wat_dir. The channels,
        subcatchments, and flowpaths args specify what slope files
        should be written.
        """
        out_dir = self.wat_dir
        if channels:
            self._make_channel_slps(out_dir, cell_width=cell_width)

        if subcatchments:
            self._write_subcatchment_slps(out_dir)

        if flowpaths:
            self._write_flowpath_slps(out_dir)

    def _make_channel_slps(self, out_dir, cell_width=None):
        ws = self.watershed
        translator = self.translator

        chn_enums = ws["channels"].keys()
        chn_enums = sorted([translator.chn_enum(chn_id=v) for v in chn_enums])

        # watershed run requires a slope file defining all of the channels in the
        # 99.1 format. Here we write a combined channel slope file and a slope
        # file for each individual channel
        fp2 = open(_join(out_dir, 'channels.slp'), 'w')
        fp2.write('99.1\n')
        fp2.write('%i\n' % len(chn_enums))

        for chn_enum in chn_enums:
            top = translator.top(chn_enum=chn_enum)
            chn_id = 'chn_%i' % top
            d = ws['channels'][chn_id]

            if cell_width is None:
                _cell_width = d.cell_width
            else:
                _cell_width = cell_width

            slp_fn = _join(out_dir, '%s.slp' % chn_id)
            fp = open(slp_fn, 'w')
            write_slp(d.aspect, d.width, _cell_width, d.length,
                      d.slopes, d.distance_p, fp, 97.3)
            fp.close()

            write_slp(d.aspect, d.width, _cell_width, d.length,
                      d.slopes, d.distance_p, fp2, 99.1)

        fp2.close()

    def _write_subcatchment_slps(self, out_dir):
        ws = self.watershed
        cellsize = self.cellsize

        for sub_id, d in ws['hillslopes'].items():
            slp_fn = _join(out_dir, '%s.slp' % sub_id)
            fp = open(slp_fn, 'w')
            write_slp(d.aspect, d.width, cellsize, d.length,
                      d.w_slopes, d.distance_p, fp, 97.3)
            fp.close()

    def _write_flowpath_slps(self, out_dir):
        ws = self.watershed
        cellsize = self.cellsize

        for sub_id in ws['hillslopes']:
            for fp_id, d in ws['flowpaths'][sub_id].items():
                fp = open(_join(out_dir, fp_id + '.slp'), 'w')
                write_slp(d.aspect, d.width, cellsize, d.length,
                          d.slopes, d.distance_p, fp, 97.3)
                fp.close()

    def px_to_utm(self, x: int, y: int) -> Tuple[float, float]:
        _transform = self.transform
        e = _transform[0] + _transform[1] * x
        n = _transform[3] + _transform[5] * y
        return e, n

    def px_to_lnglat(self, x: int, y: int) -> Tuple[float, float]:
        proj2wgs_transformer = self.proj2wgs_transformer

        e, n = self.px_to_utm(x, y)
        lng, lat = proj2wgs_transformer.transform(e, n)
        assert not np.isinf(lng), (self.transform, x, y, e, n)
        assert not np.isinf(lat)
        return lng, lat

    def _read_netw_tab(self):
        translator = self.translator

        keys = "chnum order row col row1 col1 outr outc chnlen elevvup "\
               "elevdn areaup areadn1 areadn dda node1 node2 node3 node4 "\
               "node5 node6 node7 slopedirect slopesmoothed".split()

        types = [int, int, int, int, int, int, int, int,
                 float, float, float, float, float, float,
                 float, int, int, int, int, int, int, int,
                 float, float]

        wd = self.wd
        subwta = self.subwta

        netw_tab = _join(wd, "NETW.TAB")

        with open(netw_tab) as fid:
            lines = fid.readlines()

        last = [i for i, v in enumerate(lines) if v.strip().startswith("-1")][0]
        lines = lines[27:last]
        lines = [[_t(v) for _t, v in zip(types, l.split())] for l in lines]
        assert all([len(l) == 24 for l in lines])
        data = {l[0]: dict(zip(keys, l)) for l in lines}
        # -> Dict[int, Dict[str, List[Union[int, float]]]]
        network = {}

        for chnum in data:
            head = data[chnum]["col"]-1, data[chnum]["row"]-1
            center = data[chnum]["col1"]-1, data[chnum]["row1"]-1
            tail = data[chnum]["outc"]-1, data[chnum]["outr"]-1

            chn_id0 = subwta[head]
            chn_id1 = subwta[center]
            chnout_id = subwta[tail]

            assert chn_id0 != 0, chn_id0
            assert chn_id1 != 0, chn_id1
            assert chnout_id != 0, chnout_id
            assert chn_id0 == chn_id1
            assert str(chn_id0).endswith("4"), chn_id0
            assert str(chnout_id).endswith("4"), chnout_id
            assert chn_id0 >= chnout_id, "%i %i" % (chnout_id, chn_id0)

            data[chnum]["chn_id"] = chn_id0
            data[chnum]["chnout_id"] = chnout_id

            node_indexes = [data[chnum]["node%i" % i] for i in range(1, 8)]
            node_indexes = [indx for indx in node_indexes if indx != 0]

            for chn_enum in node_indexes:
                if chn_enum > translator.channel_n:
                    continue

                chn_id = "chn_%i" % translator.top(chn_enum=chn_enum)
                if chn_id not in self.watershed["channels"]:
                    self.watershed["channels"][chn_id] = {}

                order = data[chnum]["order"]
                self.watershed["channels"][chn_id].order = order
                # self.watershed["channels"][chn_id].width = self.cellsize / order

            if chnout_id == chn_id0:
                continue

            if chnout_id not in network:
                network[chnout_id] = [chn_id0]
            else:
                network[chnout_id].append(chn_id0)

        if not all([len(L) <= 3 for L in network.values()]):
            raise ChannelRoutingError()

        self._linkdata = data
        self._network = network

    def _read_outlet(self, dnmcnt_inp) -> Tuple[int, int]:
        subwta = self.subwta

        with open(dnmcnt_inp) as fid:
            lines = fid.readlines()

            row = int(lines[98])-1
            col = int(lines[107])-1

            if not str(subwta[col, row]).endswith('4'):
                raise Exception('Identified outlet location is not a channel')

            return col, row

    def _determine_aspect(self, indx, indy) -> float:
        taspec = self.taspec
        rads = np.array(taspec[(indx, indy)]) * pi / 180.0
        return float(circmean(rads) * 180.0 / pi)

    def abstract_channels(self, wepp_chn_type='Default', verbose=False):
        subwta = self.subwta

        # extract the subcatchment and channel ids from the subwta map
        # subwta contains 0 values outside the watershed
        _ids = sorted(list(set(subwta.flatten())))

        # channels end in 4
        chn_ids = [i for i in _ids if str(i).endswith('4') and i > 0]

        n = len(chn_ids)
        if n == 0:
            raise Exception('SUBWTA contains no channels')

        for i, chn_id in enumerate(chn_ids):
            if verbose:
                print('abstracting channel %s (%i of %i)...' % (chn_id, i+1, n))
            self.abstract_channel(chn_id, wepp_chn_type=wepp_chn_type)

        self.channel_n = len(chn_ids)

    def _walk_channel(self, chn_id: int) -> \
            Tuple[np.array, np.array, np.array, np.array, np.array]:
        """
        for a channel specified by chn_id identifies
        the flowpath of the channel from top to bottom,
        the slope along the flowpath and the distance
        between points in the flowpath
        """
        subwta = self.subwta
        relief = self.relief
        flovec = self.flovec
        fvslop = self.fvslop
        paths = self.paths

        # use the subwta map to identify the coordinates the channel
        indx, indy = np.where(subwta == chn_id)

        # Relief contains the elevation values of the terrain.
        # this creates a list of tuples containing indices and elevations
        # the indices are aligned to the indx and indy arrays and are sorted
        # in descending order by elevation. We can't walk down the flovec map
        # because the length of the channel gets exaggerated due to the path
        # wandering in and out of the channel.
        indices = sorted([t for t in enumerate(relief[indx, indy])],
                         key=lambda tup: tup[1], reverse=True)

        # build the flowpath
        flowpath = [[indx[t[0]], indy[t[0]]] for t in indices]

        # follow flovec down hill an additional step to connect
        # to next channel
        c, r = flowpath[-1]
        vec = flovec[c, r]
        (x, y), dis = paths[vec]
        c += x
        r += y
        flowpath.append([c, r])
        flowpath = np.array(flowpath)

        # calculate slope and distance
        slope = np.array([fvslop[indx[t[0]], indy[t[0]]] for t in indices])
        _distance = flowpath[:-1, :] - flowpath[1:, :]
        distance = np.sqrt(np.power(_distance[:, 0], 2.0) +
                           np.power(_distance[:, 1], 2.0))

        assert distance.shape == slope.shape

        # return to _abstract_channel
        return flowpath, slope, distance, indx, indy

    def abstract_channel(self, chn_id: int, wepp_chn_type='Default'):
        """
        define channel abstraction for the purposes of running WEPP
        """
        watershed = self.watershed
        translator = self.translator
        cellsize = self.cellsize

        flowpath, slope, distance, indx, indy = self._walk_channel(chn_id)

        # need normalized distance_p to define slope
        distance_p = cummnorm_distance(distance)
        if len(slope) == 1:
            slope = np.array([float(slope), float(slope)])

        # calculate the length from the distance array
        length = float(np.sum(distance) * cellsize)
        width = float(cellsize)
        aspect = float(self._determine_aspect(indx, indy))
        isoutlet = bool(chn_id == self.outlet_top_id)

        head = [v * cellsize for v in flowpath[-1]]
        head = [float(v) for v in head]
        tail = [v * cellsize for v in flowpath[0]]
        tail = [float(v) for v in tail]

        direction = compute_direction(head, tail)

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        elevs = representative_normalized_elevations(distance_p, slope)
        slope_scalar = float(abs(elevs[-1]))

        chn_summary = ChannelSummary(
            topaz_id=chn_id,
            wepp_id=translator.wepp(top=chn_id),
            chn_enum=translator.chn_enum(top=chn_id),
            chn_type=wepp_chn_type,
            isoutlet=isoutlet,
            length=length,
            width=width,
            aspect=aspect,
            head=head,
            tail=tail,
            direction=direction,
            elevs=elevs,
            slope_scalar=slope_scalar,
            color=slp_asp_color(slope_scalar, aspect),
            area=float(length) * float(width),
            distance_p=distance_p.tolist(),
            slopes=slope.tolist(),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )

        # save channel abstraction to instance
        watershed['channels']['chn_%i' % chn_id] = chn_summary

    def abstract_subcatchments(self, verbose=False, warn=False):
        subwta = self.subwta

        # extract the subcatchment and channel ids from the subwta map
        # subwta contains 0 values outside the watershed
        _ids = sorted(list(set(subwta.flatten())))

        # subcatchments end in 1, 2, and 3, subcatchments that end in
        # are source subcatchments, 2 are left and 3 are right
        sub_ids = [i for i in _ids if not str(i).endswith('4') and i > 0]

        n = len(sub_ids)
        if n == 0:
            raise Exception('SUBWTA contains no channels')

        for i, sub_id in enumerate(sub_ids):
            if verbose:
                print('abstracting subtatchment %s (%i of %i)' % (sub_id, i+1, n))
            self.abstract_subcatchment(sub_id, verbose=verbose, warn=warn)

        self.hillslope_n = len(sub_ids)

    def _walk_flowpath(self, sub_id: int, c: int, r: int, warn=False) -> Tuple[np.array, np.array, np.array]:
        """
        walk down the gradient until we reach a channel
        to find the flowpath in pixel coords, the slope
        along the flowpath and the distance between the
        cells
        """
        subwta = self.subwta
        flopat = self.flopat
        flovec = self.flovec
        fvslop = self.fvslop
        paths = self.paths

        # we are providing the starting point of the flowpath
        flowpath = [(c, r)]
        distance = []

        n, m = subwta.shape

        # time to walk down the path
        i = 0
        while 1:
            vec = flovec[c, r]

            # break if we hit a depression, not sure what
            # happens if we hit this
            if vec == 5:
                break

            # x and y specify the direction we need to move
            (x, y), dis = paths[vec]
            c += x
            r += y

            # make sure we aren't in a loop
            if (c, r) in flowpath:
                if warn:
                    warnings.warn('Flowpath c=%i, r=%i for %i went in a circle' % (c, r, sub_id))
                break

            # store the new values
            flowpath.append((c, r))
            distance.append(dis)

            # determine whether we have reached the end of the flowpath
            _id = subwta[c, r]
            pat = flopat[c, r]
            if pat in [0, 1] or _id != sub_id:
                break

            i += 1

            if i > n + m:
                if warn:
                    warnings.warn('Flowpath c=%i, r=%i for %i is too long (>N)' % (c, r, sub_id))
                break

        # cast flowpath and distance as ndarrays
        flowpath = np.array(flowpath)
        distance = np.array(distance)

        # extract the slope values for the flowpath. We don't take the
        # last flowpath so it is aligned with the distance array
        slope = fvslop[(flowpath[:-1, 0], flowpath[:-1, 1])]

        # double check...
        assert distance.shape == slope.shape

        # return to _walk_flowpaths
        return flowpath, slope, distance

    def _walk_flowpaths(self, sub_id: int, verbose=False, warn=False) -> \
            Tuple[List[np.array], List[np.array], List[np.array], np.array, np.array]:
        """
        considers each cell of the subcatchment as a starting
        point for a flowpath. It walks down each flowpath to
        determine the slopes and distance for each.
        """
        subwta = self.subwta

        flowpaths = []
        slopes = []
        distances = []

        indx, indy = np.where(subwta == sub_id)

        n = len(indx)
        for i, (c, r) in enumerate(zip(indx, indy)):
            if verbose:
                print('walking flowpath %i of %i (%s)' % (i+1, n, sub_id))

            flowpath, slope, distance = self._walk_flowpath(sub_id, c, r, warn=warn)
            flowpaths.append(flowpath)
            slopes.append(slope)
            distances.append(distance)

        # return to abstract_subcatchment
        return flowpaths, slopes, distances, indx, indy

    def abstract_flowpath(self, flowpath, slope, distance) -> FlowpathSummary:
        cellsize = self.cellsize
        cellsize2 = self.cellsize2

        indx, indy = flowpath[:, 0], flowpath[:, 1]

        # we really only care about the longest flowpath
        # which is the first because of how we identify the
        # subflows
        distance_p = cummnorm_distance(distance)
        if len(slope) == 1:
            slope = np.array([float(slope), float(slope)])
        length = float(np.sum(distance) * cellsize)
        area = float(np.sum(distance) * cellsize2)
        width = float(area / length)
        aspect = float(self._determine_aspect(indx, indy))

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        head = [v * cellsize for v in [indx[-1], indy[-1]]]
        head = [float(v) for v in head]
        tail = [v * cellsize for v in [indx[0], indy[0]]]
        tail = [float(v) for v in tail]

        direction = compute_direction(head, tail)

        elevs = representative_normalized_elevations(distance_p, slope)
        slope_scalar = float(abs(elevs[-1]))
        color = slp_asp_color(slope_scalar, aspect)

        return FlowpathSummary(
            topaz_id=None,
            wepp_id=None,
            length=float(length),
            width=float(width),
            area=float(area),
            aspect=float(aspect),
            direction=float(direction),
            slope_scalar=slope_scalar,
            color=color,
            coords=[(int(x), int(y)) for (x, y) in zip(indx, indy)],
            distance_p=distance_p.tolist(),
            slopes=slope.tolist(),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )

    def abstract_flowpaths(self, sub_id: int, flowpaths: List[np.array],
                           slopes: List[np.array], distances: List[np.array]) -> \
            Tuple[Dict[str, FlowpathSummary], List[List[int]]]:

        # returns a list of lists
        # each list contains indices corresponding to the flowpaths that
        # follow the same route. The index of the longest flowpath is
        # listed first
        subflows = _identify_subflows(flowpaths)

        # initialize a dictionary to hold the flowpath data
        fp_d = {}
        for j, fp_indxs in enumerate(subflows):
            # build the flowpath id
            # start at 1 to make sure wepp is happy
            fp_id = 'flow_%i_%i' % (sub_id, j+1)

            fp_d[fp_id] = self.abstract_flowpath(flowpaths[fp_indxs[0]],
                                                 slopes[fp_indxs[0]],
                                                 distances[fp_indxs[0]])

        return fp_d, subflows

    def abstract_subcatchment(self, sub_id, verbose=False, warn=False):
        """
        define subcatchment abstraction for the purposes of running WEPP
        """
        watershed = self.watershed
        translator = self.translator
        cellsize2 = self.cellsize2
        cellsize = self.cellsize

        # determine flowpaths
        # Tuple[List[np.array], List[np.array], List[np.array], np.array, np.array]
        flowpaths, slopes, distances, indx, indy = \
            self._walk_flowpaths(sub_id, verbose=verbose, warn=warn)

        # determine area, width, and length
        area = float(len(indx)) * cellsize2

        # find corresponding chn_id
        chn_id = 'chn_%i' % (int(math.floor(sub_id / 10.0) * 10) + 4)
        chn_summary = watershed["channels"][chn_id]

        # If subcatchment is a source type then we calculate the distance
        # by taking a weighted average based on the length of the flowpaths
        # contained in the subcatchment
        if str(sub_id).endswith('1'):
            length = cellsize * garbrecht_length(distances)
            width = area / length

        # Otherwise the  width of the subcatchment is determined by the
        # channel that the subcatchment drains into. The length is
        # then determined by the area / width
        else:
            width = chn_summary.length
            length = area / width

        direction = chn_summary.direction
        if str(sub_id).endswith('2'):
            direction += 90
        if str(sub_id).endswith('3'):
            direction -= 90

        # determine aspect
        aspect = self._determine_aspect(indx, indy)

        # calculate weighted slope from flowpaths
        w_slopes, distance_p = \
            _weighted_slope_average(flowpaths, slopes, distances)

        # abstract the flowpaths
        fp_d, subflows = \
            self.abstract_flowpaths(sub_id, flowpaths, slopes, distances)

        watershed['flowpaths']['hill_%i' % sub_id] = fp_d

        # cast lists of ndarrays to list of lists so we can dump to json
        flowpaths = [x.tolist() for x in flowpaths]
#        distances = [x.tolist() for x in distances]
#        slopes = [x.tolist() for x in slopes]

        # determine pourpoint from flowpaths
        pourpoint = Counter([tuple(fp[-1]) for fp in flowpaths]).most_common()[0][0]
        pourpoint = list([v * cellsize for v in pourpoint])

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        elevs = representative_normalized_elevations(distance_p, w_slopes)
        slope_scalar = float(abs(elevs[-1]))

        sub_summary = HillSummary(
            topaz_id=sub_id,
            wepp_id=translator.wepp(top=sub_id),
            w_slopes=list(w_slopes),
            pourpoint=list(pourpoint),
            length=float(length),
            width=float(width),
            area=float(area),
            aspect=float(aspect),
            direction=direction,
            slope_scalar=slope_scalar,
            color=slp_asp_color(slope_scalar, aspect),
            distance_p=list(distance_p),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )
        watershed['hillslopes']['hill_%i' % sub_id] = sub_summary

    def abstract_structure(self, verbose=False):
        self._read_netw_tab()
        network = self.network
        translator = self.translator

        # now we are going to define the lines of the structure file
        # this doesn't handle impoundments

        structure = []
        for chn_id in translator.iter_chn_ids():
            if verbose:
                print('abstracting structure for channel %s...' % chn_id)
            top = translator.top(chn_id=chn_id)
            chn_enum = translator.chn_enum(chn_id=chn_id)

            # right subcatchments end in 2
            hright = top - 2
            if not translator.has_top(hright):
                hright = 0

            # left subcatchments end in 3
            hleft = top - 1
            if not translator.has_top(hleft):
                hleft = 0

            # center subcatchments end in 1
            hcenter = top - 3
            if not translator.has_top(hcenter):
                hcenter = 0

            # define structure for channel
            # the first item defines the channel
            _structure = [chn_enum]

            # network is defined from the NETW.TAB file that has
            # already been read into {network}
            # the 0s are appended to make sure it has a length of
            # at least 3
            chns = network.get(top, []) + [0, 0, 0]

            # structure line with top ids
            _structure += [hright, hleft, hcenter] + chns[:3]

            # this is where we would handle impoundments
            # for now no impoundments are assumed
            _structure += [0, 0, 0]

            # and translate topaz to wepp
            structure.append([int(v) for v in _structure])

        self._structure = structure

if __name__ == '__main__':
    watershed_a = WatershedAbstraction('/geodata/weppcloud_runs/419e3b7f-ae62-4bc1-af10-bba44266f494/dem/topaz',
                                       '/geodata/weppcloud_runs/419e3b7f-ae62-4bc1-af10-bba44266f494/watershed')
    watershed_a.abstract(verbose=False)
#    watershed_a.write_slps(channels=1, subcatchments=1, flowpaths=1)
#    js = watershed_a.json()

#    watershed_b = WatershedAbstraction()
#    watershed_b.load(js)
#    watershed_b.dumpjson('wb.json')
#    pprint(watershed_b.watershed["channels"])


p = []
with open("inp.txt", "r") as f:
    for line in f:
        p.append(int(line.strip()))

# p=[100756]
r=0
for i in p:
    s = 0
    while i >= 0:
        i = i // 3 - 2
        if i >= 0:
            s += i
    r += s
print(r)


from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from functools import reduce
from utils import clean_line, get_top_matches
import re

def strip_page_stop_words(s):
    tokens = word_tokenize(s)
    return [word for word in tokens if word not in stopwords.words()]

def scrape_dissertation_abstract(page, alumni_list, faculty_list):
    if page['text'] == 'ERROR':
        return {'alumni_match': None,
                'alumni_compared_line': None,
                'faculty_matches': None
                }

    page = [line for line in re.split('; |, |\*|\n', page['text']) if line not in ['', None]]

    cleaned_lines = []
    for line in page:
        line = strip_page_stop_words(line)
        if len(line) > 0:
            line = reduce(lambda a, b: a + ' ' + b, line)
            cleaned_lines.append(line)

    page = cleaned_lines
    committee_index = next((i for i, line in enumerate(page) if 'committee' in clean_line(line)), 0)

    alumni_matches = get_top_matches(page, alumni_list, normalize_func=lambda i: i)
    faculty_matches = get_top_matches(page[committee_index + 1:], faculty_list, normalize_func=lambda i: i)

    alumni_matches = [tup for tup in alumni_matches if tup[2] >= 0.90]
    faculty_matches = [tup for tup in faculty_matches if tup[2] >= 0.90]

    if len(alumni_matches):
        compared_line = reduce(lambda a, b: str(a) + ' ' + str(b),
                               [tup[0]
                                for tup
                                in alumni_matches
                                if tup[2] >= 0.90
                                ]
                               )

        alumni_matches = [tup[1]
                          for tup
                          in alumni_matches
                          ]
    else:
        compared_line = None
        alumni_matches = None

    if len(faculty_matches):
        faculty_matches = [[tup[1], 'Chair']
                           if (i == 0) or ('chair' in tup[0].lower())
                           else [tup[1], 'Non-chair']
                           for i, tup
                           in enumerate(faculty_matches)
                           ]
    else:
        faculty_matches = None

    return {'alumni_match': alumni_matches,
            'alumni_compared_line': compared_line,
            'faculty_matches': faculty_matches
            }

def scrape_dissertation_abstracts(page_list, alumni_list, faculty_list):
    for i, page in enumerate(page_list):
        page = {**page, **scrape_dissertation_abstract(page, alumni_list, faculty_list)}
        page_list[i] = page
    return page_list


# Manipulation of a dictionary

def dir_manipulation(dir):
    # deletes two keys from the dictionary
    del zoo_animals['Sloth']
    del zoo_animals['Bengal Tiger']

    # edits the value of a key
    zoo_animals['Rockhopper Penguin'] = 'Jungle House'

    return zoo_animals

if __name__ == '__main__':

    print("Exercise 2")

    zoo_animals = {
        'Unicorn': 'Cotton Candy House',
        'Sloth': 'Rainforest Exhibit',
        'Bengal Tiger': 'Jungle House',
        'Atlantic Puffin': 'Arctic Exhibit',
        'Rockhopper Penguin': 'Arctic Exhibit'
    }

    print("Original dictionary")
    for key, value in zoo_animals.items():
        print(key, value)

    print("\nNew dictionary")
    for key, value in dir_manipulation(zoo_animals).items():
        print(key, value)


from typing import Any, Dict, List, Text

import regex
import re
from unicodedata import normalize as nl

import rasa.shared.utils.io
import rasa.utils.io
from rasa.shared.constants import DOCS_URL_COMPONENTS
from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
from rasa.shared.nlu.training_data.message import Message
from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
from rasa.shared.nlu.constants import (
    TEXT,
    INTENT,
    RESPONSE_IDENTIFIER_DELIMITER,
    ACTION_NAME,
)

class VietnameseTokenizer(Tokenizer):

    defaults = {
        # Flag to check whether to split intents
        "intent_tokenization_flag": False,
        # Symbol on which intent should be split
        "intent_split_symbol": "_",
        # Regular expression to detect tokens
        "token_pattern": None,
    }

    # the following language should not be tokenized using the VietnameseTokenizer
    not_supported_language_list = ["zh", "ja", "th"]

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
        """Construct a new tokenizer using the VietnameseTokenizer framework."""

        super().__init__(component_config)

        self.emoji_pattern = rasa.utils.io.get_emoji_regex()
        self.text_normalized = ""

        if "case_sensitive" in self.component_config:
            rasa.shared.utils.io.raise_warning(
                "The option 'case_sensitive' was moved from the tokenizers to the "
                "featurizers.",
                docs=DOCS_URL_COMPONENTS,
            )

    def remove_emoji(self, text: Text) -> Text:
        """Remove emoji if the full text, aka token, matches the emoji regex."""
        match = self.emoji_pattern.fullmatch(text)

        if match is not None:
            return ""

        return text

    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
        text = message.get(attribute)
        text = text.strip()
        text = text.strip(u"\ufeff")
        text = text.strip(u"\u200b\u200b\u200b\u200b\u200b\u200b\u200b")
        text = nl('NFKC', text)
        self.text_normalized = self.text_normalize(text.lower())

        # we need to use regex instead of re, because of
        # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters

        # remove 'not a word character' if
        words = regex.sub(
            # there is a space or an end of a string after it
            r"[^\w#@&]+(?=\s|$)|"
            # there is a space or beginning of a string before it
            # not followed by a number
            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
            # not in between numbers and not . or @ or & or - or #
            # e.g. 10'000.00 or blabla@gmail.com
            # and not url characters
            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
            " ",
            self.text_normalized,
        ).split()

        words = [self.remove_emoji(w) for w in words]
        words = [w for w in words if w]

        # if we removed everything like smiles `:)`, use the whole text as 1 token
        if not words:
            words = [self.text_normalized]

        tokens = self._convert_words_to_tokens(words, self.text_normalized)

        return self._apply_token_pattern(tokens)

    def process(self, message: Message, **kwargs: Any) -> None:
        """Tokenize the incoming message."""
        for attribute in MESSAGE_ATTRIBUTES:
            if isinstance(message.get(attribute), str):
                if attribute in [INTENT, ACTION_NAME, RESPONSE_IDENTIFIER_DELIMITER]:
                    tokens = self._split_name(message, attribute)
                else:
                    tokens = self.tokenize(message, attribute)

                message.set(TOKENS_NAMES[attribute], tokens)
        message.set(TEXT, self.text_normalized, add_to_output=True)

    def text_normalize(self, text):
        """
        Normalize Vietnamese accents
        """

        text = re.sub(r"òa", "oà", text)
        text = re.sub(r"óa", "oá", text)
        text = re.sub(r"ỏa", "oả", text)
        text = re.sub(r"õa", "oã", text)
        text = re.sub(r"ọa", "oạ", text)
        text = re.sub(r"òe", "oè", text)
        text = re.sub(r"óe", "oé", text)
        text = re.sub(r"ỏe", "oẻ", text)
        text = re.sub(r"õe", "oẽ", text)
        text = re.sub(r"ọe", "oẹ", text)
        text = re.sub(r"ùy", "uỳ", text)
        text = re.sub(r"úy", "uý", text)
        text = re.sub(r"ủy", "uỷ", text)
        text = re.sub(r"ũy", "uỹ", text)
        text = re.sub(r"ụy", "uỵ", text)
        text = re.sub(r"Ủy", "Uỷ", text)

        return text


#!/usr/bin/env python
#encoding:utf-8

# 最最原始的一个Web框架;

import socket

'''
对于web框架而言；
	服务器程序一般相同;
	应用程序一般不同;
那么服务器程序和应用程序之间需要一个标准，就是wsgi(web server gateway interface)，实现web app和web server的解耦。这时服务器程序只需要一个，
应用程序可以根据需求进行变化.
'''

# 应用程序
def handle_request(client):
	buf = client.recv(1024)
	client.send("HTTP/1.1 OK\r\n\r\n")
	client.send("Hello westos")

# 服务器程序
def main():
	sock = socket.socket()
	sock.bind(('localhost', 8888))
	sock.listen(5)

	while True:
		connection, address = sock.accept()
		handle_request(connection)
		connection.close()
if __name__ == "__main__":
	main()


import csv
import numpy as np

# from sentence_embedding import sent_to_vec

def abcd_valid_extended(row):
    """Takes a list of 4 strings as input,
        embedd each sentence then extends it to 8 valid analogies

    Args:
        row (list): a list of 4 strings to embedd
        embedding_dict (str, optional): the ebedding dictionnary used. Defaults to "../data/glove.6B/".
        embedding_size (int, optional): Size of the embedded word vector. Defaults to 100.

    Returns:
        list: a list containing the 8 or 4 valid analogies sentence lists
    """
    a = row[0]
    b = row[1]
    c = row[2]
    d = row[3]
    return [
        np.stack([a, b, c, d]).T, #
        np.stack([b, a, d, c]).T, #
        np.stack([c, d, a, b]).T, #
        np.stack([d, c, b, a]).T, #
        # np.stack([a, c, b, d]).T,
        # np.stack([d, b, c, a]).T,
        # np.stack([c, a, d, b]).T,
        # np.stack([b, d, a, c]).T
    ]

def bacd_invalid_extended(row):
    a = row[0]
    b = row[1]
    c = row[2]
    d = row[3]
    return abcd_valid_extended([b, a, c, d])

def cbad_invalid_extended(row):
    a = row[0]
    b = row[1]
    c = row[2]
    d = row[3]
    return abcd_valid_extended([c, b, a, d])


import numpy as np
import config
import glob
import read_tiff
from imageio import imread
import h5py

h5s = {str(fn): h5py.File(config.data_path + str(fn) + ".h5", "a")
       for fn in range(1, 6)}

for full_fname in glob.glob(config.annotations_path_c + "*.png"):
    fname = full_fname.rsplit("/", 1)[1]
    h5 = h5s[fname[0]]
    fname = fname[4:10]
    fname = config.c_anno_groupname + fname
    if fname in h5:
        # continue
        h5.__delitem__(fname)
    print(fname, h5)
    image = imread(full_fname)
    annotation = read_tiff.get_mask(image)
    print(np.unique(annotation, return_counts=True))
    h5.create_dataset(fname, data=annotation)


#!/usr/bin/env python2

"""
"""

import argparse
import csv
import errno
import os
import sys

class Vopitem(object):
    """_voted attribs are None or a vote count (potentially zero)"""

    def __init__(self, contest, choice, pct_id, barcode=None,
                 voph=None, voth=None, vopnt=None, votnt=None, vopot=None, votot=None):

        self.contest = contest
        self.choice = choice
        self.pct_id = pct_id
        self.voph = voph        # voteop HARTS
        self.voth = voth        # voted  HARTS
        self.vopnt = vopnt      # voteop new TEVS
        self.votnt = votnt      # voted  new TEVS
        self.vopot = vopot      # voteop old TEVS
        self.votot = votot      # voted  old TEVS
        # assert not barcode is None and  barcode.isdigits()
        self.barcode = barcode

    def __str__(self):
        s = '<Vopitem:'
        vs = ['{}: {}'.format(k, self.__dict__[k]) for k in sorted(self.__dict__.keys())]
        s += ', '.join(vs)
        return s + '>'

    def liststr(self):
        return  [self.contest, self.choice, self.pct_id, self.barcode,
                str(self.voph), str(self.voth), str(self.vopnt), str(self.votnt),
                str(self.vopot), str(self.votot)]

class Vops(dict):

    def __init__(self):
        super(Vops, Vops).__init__(self)

    def upd_rec_old_tevs (self, contest, choice, pct_id, barcode, voteops, votes):
        key = (contest, choice, pct_id)
        item = self.get(key, Vopitem(contest, choice, pct_id))
        item.vopot = add_to_none(item.vopot, voteops)
        item.votot = add_to_none(item.votot, votes)
        if barcode and barcode != '--na--':
            item.barcode = barcode
        self[key] = item

    def upd_rec_new_tevs (self, contest, choice, pct_id, barcode, voteops, votes):
        key = (contest, choice, pct_id)
        item = self.get(key, Vopitem(contest, choice, pct_id))
        item.vopnt = add_to_none(item.vopnt, voteops)
        item.votnt = add_to_none(item.votnt, votes)
        item.barcode = barcode
        self[key] = item

    def upd_rec_harts(self, contest, choice, pct_id, voteops, votes):
        key = (contest, choice, pct_id)
        item = self.get(key, Vopitem(*key))
        item.voph = add_to_none(item.voph, voteops)
        item.voth = add_to_none(item.voth, votes)
        self[key] = item

    def iter_rec(self):
        for key in sorted(self.keys()):
            yield self[key]

class Vopstots(dict):
    """Totals across precincts."""

    def __init__(self):
        super(Vopstots, Vopstots).__init__(self)

    def upd_rec(self, contest, choice, voph, voth, votot, vopnt, votnt):
        key = (contest, choice)
        item = self.get(key, Vopitem(contest, choice, None))
        item.voph = add_to_none(item.voph, voph)
        item.voth = add_to_none(item.voth, voth)
        item.votot = add_to_none(item.votot, votot)
        item.vopnt = add_to_none(item.vopnt, vopnt)
        item.votnt = add_to_none(item.votnt, votnt)
        self[key] = item

    def iter_rec(self):
        for key in sorted(self.keys()):
            yield self[key]

def add_to_none(target, value):
    v1 = target if target else 0
    v2 = value  if value else 0
    return v1 + v2

def diff(x, y):
    if x is None:
        return y
    if y is None:
        return x
    return x - y

def get_clargs():
    argpars = argparse.ArgumentParser(description='Compare TEVS output to HARTS',
                                      prog=os.path.basename(__file__))
    argpars.add_argument('--progress', type=int, default=None,
                         help='report progress to stderr every N lines')
    argpars.add_argument('infile', nargs='?', type=argparse.FileType('r'),
        default = sys.stdin, help='HARTS consolidated counts')
    argpars.add_argument('--harts', type=argparse.FileType('r'),
        help='TEVS consolidated counts')
    argpars.add_argument('--oldtevs', type=argparse.FileType('r'),
        help='report with old tevs counts')
    argpars.add_argument('--rptdir', default='.',
        help='directory where report_pct.tsv and report_overall.tsv will be written, default "."')
    return argpars.parse_args()

# ---------------------------------------- MAIN ---------------------------------------- #

clargs = get_clargs()
stderr = sys.stderr

rptdir = clargs.rptdir

try:
    os.makedirs(rptdir)
except OSError as e:
    if not os.path.isdir(rptdir):
        raise

rptp = os.path.join(rptdir, 'report_both_pct.tsv')              # output
rpto = os.path.join(rptdir, 'report_both_overall.tsv')          # output
oldtevsf = clargs.oldtevs

lcnt = 0
vops = Vops()

if clargs.progress:
    stderr.write('PROCESSING TEVS NEW\n')

with clargs.infile as inf:
    for line in inf:
        lcnt += 1
        fields = line.split('\t')
        f = [x.strip() for x in fields]
        contest, choice, pct, barcode, voteops, votes = \
            f[0], f[1], f[2], f[3], int(f[4]), int(f[5])
        vops.upd_rec_new_tevs (contest, choice, pct, barcode, voteops, votes)

if clargs.progress:
    stderr.write('PROCESSING OLD TEVS\n')
with oldtevsf as newtf:
    for r in csv.DictReader(oldtevsf, dialect=csv.excel_tab):
        r['Choice'] = r['Choice'].upper()
        # r['Choice'] = r['Choice'].replace('"', '')
        if r['Contest'] + r['Choice'] + r['Precinct'] == '':
            continue
        contest, choice, pct, barcode, votes = \
            r['Contest'], r['Choice'], r['Precinct'][3:], r['template'], int(r['TEVS Count'])
        vops.upd_rec_old_tevs (contest, choice, pct, barcode, None, votes)

if clargs.progress:
    stderr.write('PROCESSING HARTS\n')

with clargs.harts as inf:
    for line in inf:
        lcnt += 1
        # line = line.replace('"', '')
        line = line.replace('Write-In', 'WRITE-IN')
        fields = line.split('\t')
        f = [x.strip() for x in fields]
        contest, choice, pct, voteops, votes = f[1], f[2], f[0], int(f[3]), int(f[4])
        vops.upd_rec_harts(contest, choice, pct, voteops, votes)

# emit comparison by precinct
#
with open(rptp, 'w') as ouf:
    headitems = ['contest', 'choice', 'precinct', 'barcode', 'HARTS Vote Ops', 'New TEVS Vote Ops',
                 'HARTS Votes', 'New TEVS Votes', 'Old TEVS Votes', 'Vote Ops HARTS v New TEVS',
                 'Votes HARTS v New TEVS', 'Votes HARTS v Old TEVS', 'Votes New TEVS v Old TEVS']
    headline = '\t'.join(headitems) + '\n'
    ouf.write(headline)
    for v in vops.iter_rec():
        outv = [v.contest, v.choice, v.pct_id, v.barcode, v.voph, v.vopnt,
                v.voth, v.votnt, v.votot,
                diff(v.voph, v.vopnt), diff(v.voth, v.votnt),
                diff(v.voth, v.votot), diff(v.votnt, v.votot)]
        outv = [str(x) for x in outv]
        outv[2] = '~  ' + outv[2]     # keep Excel from turning pct_id into a floating point number
        ouf.write('\t'.join(outv) + '\n')

# compute totals for entire election
#
vopstots = Vopstots()
for v in vops.iter_rec():
    vopstots.upd_rec(v.contest, v.choice, v.voph, v.voth, v.votot, v.vopnt, v.votnt)

# compute margins & tack onto vopitems in vopstots
#
contests = set((v.contest for v in vopstots.iter_rec()))
margins = dict()                    # margin for a contest across all precincts
for contest in contests:
    choices = [v for v in vopstots.iter_rec() if v.contest == contest]
    choices = sorted(choices, key=lambda v: v.voth, reverse=True)
    margins[contest] = choices[0].voth - choices[1].voth

# emit county-wide totals
#

with open(rpto, 'w') as ouf:
    headitems = ['contest', 'choice', 'HARTS Vote Ops', 'New TEVS Vote Ops', 'HARTS Votes', \
                'New TEVS Votes', 'Old TEVS Votes', 'Vote Ops HARTS v New TEVS',
                 'Votes HARTS v New TEVS', 'Votes HARTS v Old TEVS', 'Votes New TEVS v Old TEVS',
                 '% of Margin']
    headline = '\t'.join(headitems) + '\n'
    ouf.write(headline)
    for v in vopstots.iter_rec():
        vop_hart_v_newtevs = diff(v.voph, v.vopnt)
        vot_hart_v_newtevs = diff(v.voth, v.votnt)
        vot_hart_v_oldtevs = diff(v.voth, v.votot)
        vot_newtevs_v_oldtevs = diff(v.votnt, v.votot)
        percent_margin = float(vot_hart_v_newtevs) / margins[v.contest]

        outv = [v.contest, v.choice, v.voph, v.vopnt, v.voth, v.votnt, v.votot,
                vop_hart_v_newtevs, vot_hart_v_newtevs, vot_hart_v_oldtevs,
                vot_newtevs_v_oldtevs, percent_margin
               ]
        outv = [str(x) for x in outv]
        ouf.write('\t'.join(outv) + '\n')


######################### 01 #########################

def do_list_of_domains(file_name: str) -> list:
    with open(filename, 'r') as txt_file:
        result = []

        for line in txt_file.readlines():
            result.append(line[1:].strip())

    return result

filename = 'domains.txt'

result_list = do_list_of_domains(filename)

######################### 02 #########################

def do_list_of_names(file_name: str) -> list:
    with open(filename, 'r') as txt_file:
        result = []

        for line in txt_file.readlines():
            result.append(line.split('\t')[1])

    return result

filename = 'names.txt'

result_list = do_list_of_names(filename)

######################### 03 #########################
# решил не искать легких путей типа datetime

def search_and_change_date_format(file_name: str) -> list:
    res_list = []

    with open(filename, 'r') as txt_file:
        for line in txt_file.readlines():
            if line[0].isdigit():
                old_date = line.split(' - ')[0]

                # определяем dd, mm, yy для нового формата даты dd/mm/yyyy
                new_dd = old_date.split(' ')[0][:-2:]
                new_dd = '0' + new_dd if len(new_dd) == 1 else new_dd
                new_mm = month_dict.get(old_date.split(' ')[1])
                new_yy = old_date.split(' ')[2]

                new_date = (new_dd + '/' + new_mm + '/' + new_yy)
                res_list.append({"date_original": old_date, "date_modified": new_date})

    return res_list

month_dict = {'January': '01', 'February': '02', 'March': '03',
              'April': '04', 'May': '05', 'June': '06',
              'July': '07', 'August': '08', 'September': '09',
              'October': '10', 'November': '11', 'December': '12'}

filename = 'authors.txt'
result_list = search_and_change_date_format(filename)


import pygame
display_width = 1600
display_height = 900

class Item:
    def __init__(self, type, stat, x, y, itemImg, itemImgRight):
        self.type = type
        self.stat = stat
        self.x = x
        self.y = y
        self.itemImg = itemImg
        self.itemImgRight = itemImgRight
        self.imgW = 50
        self.imgH = 50

class Chest:
    def __init__(self, x, y, items):
        self.x = x
        self.y = y
        self.items = items
        self.chestImg = pygame.image.load('art/chest.png')

class Door:
    def __init__(self, combo, number, destinationNum):
        self.combo = combo
        if self.combo == ' ':
            self.locked = False
        else:
            self.locked = True
        self.doorFlash = False
        self.number = number
        self.flashTimer = 0
        self.destination = destinationNum
        self.doorBuffer = ""
        self.failedAttempt = False
        if self.number == 1:
            self.x = display_width - 50
            self.y = display_height/2 - 150
        elif self.number == 2:
            self.x = display_width/2 - 150
            self.y = display_height - 50
        elif self.number == 3:
            self.x = 0
            self.y = display_height / 2 - 150
        elif self.number == 4:
            self.x = display_width/2 - 150
            self.y = 0
        if number % 2 == 0:
            self.width = 300
            self.height = 50
            if combo == ' ':
                self.doorImg = pygame.image.load('art/door0.png')
            else:
                self.doorImg = pygame.image.load('art/door0.jpg')
        else:
            self.width = 50
            self.height = 300
            if combo == ' ':
                self.doorImg = pygame.image.load('art/door.png')
            else:
                self.doorImg = pygame.image.load('art/door.jpg')

    def canPass(self):
        if self.combo == ' ':
            self.locked = False
            return True
        return not self.locked

    def unlock(self, combo):
        if combo == self.combo:
            self.locked = False
            if self.number % 2 == 0:
                self.doorImg = pygame.image.load('art/door0.png')
            else:
                self.doorImg = pygame.image.load('art/door.png')
            return True
        else:
            return False

    def addToDoorBuffer(self, key):
        self.doorBuffer += key
        if len(self.doorBuffer) == len(self.combo) and self.locked:
            if not self.unlock(self.doorBuffer):
                if self.number % 2 == 0:
                    self.doorImg = pygame.image.load('art/locked0.png')
                    self.failedAttempt = True
                else:
                    self.doorImg = pygame.image.load('art/locked.png')
                    self.failedAttempt = True
            self.doorBuffer = ""

class Chamber:
    def __init__(self, name, doors):
        self.chests = []
        self.npcs = []
        self.tidas = []
        self.items = []
        self.name = name
        self.doors = doors

    def addChest(self, chest):
        self.chests.append(chest)

    def addNpc(self, npc):
        self.npcs.append(npc)

    def addTida(self, tida):
        self.tidas.append(tida)

class Graph:
    def __init__(self, size):
        self.chambers = [0] * size

    def addChamber(self, chamber):
        self.chambers[chamber.name] = chamber


#!/System/Library/Frameworks/Python.framework/Versions/Current/bin/python
# -*- coding: utf-8 -*-
"""
precache.py is a tool that can be used to cache various OTA updates for
iOS/tvOS/watchOS devices, as well as Mac App Store apps, macOS Installers, and
various macOS related software updates.

For more information: https://github.com/krypted/precache
For usage: ./precache.py --help

Issues: Please log an issue via github, or fork and create a pull request with
your fix.
"""

import argparse
import base64
import sys
from precache import precache

class SaneUsageFormat(argparse.HelpFormatter):
    """
    Matt Wilkie on SO
    http://stackoverflow.com/questions/9642692/
    argparse-help-without-duplicate-allcaps/9643162#9643162
    """
    def _format_action_invocation(self, action):
        if not action.option_strings:
            default = self._get_default_metavar_for_positional(action)
            metavar, = self._metavar_formatter(action, default)(1)
            return metavar

        else:
            parts = []

            # if the Optional doesn't take a value, format is:
            #    -s, --long
            if action.nargs == 0:
                parts.extend(action.option_strings)

            # if the Optional takes a value, format is:
            #    -s ARGS, --long ARGS
            else:
                default = self._get_default_metavar_for_optional(action)
                args_string = self._format_args(action, default)
                for option_string in action.option_strings:
                    parts.append(option_string)

                return "%s %s" % (", ".join(parts), args_string)

            return ", ".join(parts)

    def _get_default_metavar_for_optional(self, action):
        return action.dest.upper()

def main():
    parser = argparse.ArgumentParser(formatter_class=SaneUsageFormat)

    asset_groups = ["AppleTV", "iPad", "iPhone", "iPod",
                    "Watch", "app", "installer", "updates"]
    ipsw_groups = ["AppleTV", "iPad", "iPhone", "iPod", "Watch"]

    asset_groups.sort()
    ipsw_groups.sort()

    parser.add_argument("--cache-group",
                        type=str,
                        nargs="+",
                        dest="cache_group",
                        choices=(asset_groups),
                        metavar="<product name>",
                        help="Cache assets based on group",
                        required=False)

    parser.add_argument("--cache-ipsw-group",
                        type=str,
                        nargs="+",
                        dest="cache_ipsw_group",
                        choices=(ipsw_groups),
                        metavar="<product name>",
                        help="Cache IPSW based on group",
                        required=False)

    parser.add_argument("-cs", "--cache-server",
                        type=str,
                        nargs=1,
                        dest="cache_server",
                        metavar="http://cacheserver:port",
                        help="Specify the cache server to use.",
                        required=False)

    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="Debug mode - increased log verbosity.",
                        required=False)

    parser.add_argument("-n", "--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Shows what would be cached.",
                        required=False)

    parser.add_argument("--filter-group",
                        type=str,
                        nargs="+",
                        dest="filter_group",
                        choices=(asset_groups),
                        metavar="<product name>",
                        help="Filter based on group",
                        required=False)

    parser.add_argument("-i", "--ipsw",
                        type=str,
                        nargs="+",
                        dest="ipsw_model",
                        metavar="model",
                        help="Cache IPSW for provided model/s.",
                        required=False)

    parser.add_argument("-l", "--list",
                        action="store_true",
                        dest="list_models",
                        help="Lists all assets available for caching.",
                        required=False)

    parser.add_argument("-m", "--model",
                        type=str,
                        nargs="+",
                        dest="model",
                        metavar="model",
                        help="Provide model(s)/app(s), i.e iPhone8,2 Xcode.",
                        required=False)

    parser.add_argument("--jamfserver",
                        type=str,
                        dest="jamfserver",
                        help="Jamf server address",
                        required=False)

    parser.add_argument("--jamfuser",
                        type=str,
                        dest="jamfuser",
                        help="Jamf server username",
                        required=False)

    parser.add_argument("--jamfpassword",
                        type=str,
                        dest="jamfpassword",
                        help="Jamf server password",
                        required=False)

    parser.add_argument("-o", "--output",
                        type=str,
                        nargs=1,
                        dest="output_dir",
                        metavar="file path",
                        help="Path to save IPSW files to.",
                        required=False)

    parser.add_argument("--version",
                        action="store_true",
                        dest="ver",
                        help="Version info.",
                        required=False)

    args = parser.parse_args()

    # While argsparse is pretty cool, it does have limits when it comes to
    # handling having items mutually exclusive against a specifc argument so
    # here this explicitly checks if args.list_models is being called and if so
    # tests if it's being passed or not.
    if len(sys.argv) > 1:
        if args.ver:
            precache.print_version()
            sys.exit(0)

        if args.list_models and (args.model or args.ipsw_model or args.ver or
                                 args.cache_group or args.cache_ipsw_group or
                                 (args.jamfserver and args.jamfuser and
                                  args.jamfpassword)):
            print("Cannot combine these arguments with -l, precache.--list.")
            sys.exit(1)
        else:
            if args.debug:
                level = "debug"
            else:
                level = "info"

            if args.dry_run:
                dry = True
            else:
                dry = False

            if args.output_dir:
                download_dir = args.output_dir[0]
            else:
                download_dir = None

            if args.cache_server:
                cache_srv = args.cache_server[0]
                p = precache.PreCache(
                    cache_server=cache_srv, log_level=level, dry_run=dry)
            else:
                p = precache.PreCache(
                    cache_server=None, log_level=level, dry_run=dry)

            if args.list_models:
                if args.filter_group:
                    p.list_assets(group=args.filter_group)
                else:
                    p.list_assets()

            if args.jamfserver and args.jamfuser and args.jamfpassword:
                from precache import jamf
                models = jamf.JamfRequest(args.jamfserver).mobile_ids(
                    args.jamfuser, args.jamfpassword)
                if models:
                    p.cache_assets(model=models)
                else:
                    logger.error("Empty models list from jamf")

            if args.model:
                p.cache_assets(model=args.model)

            if args.cache_group:
                p.cache_assets(group=args.cache_group)

            if args.cache_ipsw_group:
                p.cache_ipsw(group=args.cache_ipsw_group,
                             store_in=download_dir)

            if args.ipsw_model:
                p.cache_ipsw(model=args.ipsw_model, store_in=download_dir)
    else:
        print("%s --help for usage" % sys.argv[0])

if __name__ == "__main__":
    main()


"""final_project URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin

from network.views import (UserLoginView, UserLogoutView, UserMainPageView, AllBooksView, SpecificBookView,
                           ReserveBookView, AddBookView)

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', UserLoginView.as_view(), name="main-page"),
    url(r'^logout', UserLogoutView.as_view(), name="logout"),
    url(r'^logged-user/', UserMainPageView.as_view(), name="logged-user"),
    url(r'^all-books/$', AllBooksView.as_view(), name="all-books"),
    url(r'^all-books/(?P<book_id>(\d)+)', SpecificBookView.as_view(), name="specific-book"),
    url(r'^all-books/reserve-book/(?P<book_id>(\d)+)/(?P<owner_id>(\d)+)', ReserveBookView.as_view(), name="reserve-book"),
    url(r'^add-book/', AddBookView.as_view(), name="add-book"),
]


#!/usr/bin/env python3

# Copyright (c) 2020 Mellanox Technologies, Inc.  All rights reserved.
#
# This software is available to you under a choice of one of two
# licenses.  You may choose to be licensed under the terms of the GNU
# General Public License (GPL) Version 2, available from the file
# COPYING in the main directory of this source tree, or the
# OpenIB.org BSD license below:
#
#     Redistribution and use in source and binary forms, with or
#     without modification, are permitted provided that the following
#     conditions are met:
#
#      - Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      - Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials
#        provided with the distribution.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Parse tool for SW steering debug dump file
import argparse
import csv

from src import dr_trigger
from src.dr_action import *
from src.dr_domain import *
from src.dr_matcher import *
from src.dr_rule import *
from src.dr_table import *
from src.dr_utilities import *
from src.dr_constants import *

# mapping csv records types to it's relevant parser function
switch_csv_rec_type = {
    DR_DUMP_REC_TYPE_DOMAIN: dr_dump_domain,
    DR_DUMP_REC_TYPE_DOMAIN_INFO_DEV_ATTR: dr_dump_domain_info_dev_attr,
    DR_DUMP_REC_TYPE_DOMAIN_INFO_CAPS: dr_dump_domain_info_caps,
    DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT: dr_dump_domain_info_vport,
    DR_DUMP_REC_TYPE_DOMAIN_SEND_RING: dr_dump_domain_send_ring,
    DR_DUMP_REC_TYPE_DOMAIN_INFO_FLEX_PARSER: dr_dump_domain_info_flex_parser,

    DR_DUMP_REC_TYPE_TABLE: dr_dump_table,
    DR_DUMP_REC_TYPE_TABLE_RX: dr_dump_table_rx_tx,
    DR_DUMP_REC_TYPE_TABLE_TX: dr_dump_table_rx_tx,

    DR_DUMP_REC_TYPE_MATCHER: dr_dump_matcher,
    DR_DUMP_REC_TYPE_MATCHER_MASK: dr_dump_matcher_mask,
    DR_DUMP_REC_TYPE_MATCHER_MASK_WITH_RESERVED: dr_dump_matcher_mask,
    DR_DUMP_REC_TYPE_MATCHER_RX: dr_dump_matcher_rx_tx,
    DR_DUMP_REC_TYPE_MATCHER_TX: dr_dump_matcher_rx_tx,
    DR_DUMP_REC_TYPE_MATCHER_BUILDER: dr_dump_matcher_builder,

    DR_DUMP_REC_TYPE_RULE: dr_dump_rule,
    DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V0: dr_dump_rule_entry_rx_tx,
    DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V0: dr_dump_rule_entry_rx_tx,
    DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V1: dr_dump_rule_entry_rx_tx,
    DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V1: dr_dump_rule_entry_rx_tx,

    DR_DUMP_REC_TYPE_ACTION_ENCAP_L2: dr_dump_action_encap_l2,
    DR_DUMP_REC_TYPE_ACTION_ENCAP_L3: dr_dump_action_encap_l3,
    DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR: dr_dump_action_modify_header,
    DR_DUMP_REC_TYPE_ACTION_DROP: dr_dump_action_drop,
    DR_DUMP_REC_TYPE_ACTION_QP: dr_dump_action_qp,
    DR_DUMP_REC_TYPE_ACTION_FT: dr_dump_action_ft,
    DR_DUMP_REC_TYPE_ACTION_CTR: dr_dump_action_ctr,
    DR_DUMP_REC_TYPE_ACTION_TAG: dr_dump_action_tag,
    DR_DUMP_REC_TYPE_ACTION_VPORT: dr_dump_action_vport,
    DR_DUMP_REC_TYPE_ACTION_DECAP_L2: dr_dump_action_decap_l2,
    DR_DUMP_REC_TYPE_ACTION_DECAP_L3: dr_dump_action_decap_l3,
    DR_DUMP_REC_TYPE_ACTION_DEVX_TIR: dr_dump_action_devx_tir,
    DR_DUMP_REC_TYPE_ACTION_POP_VLAN: dr_dump_action_pop_vlan,
    DR_DUMP_REC_TYPE_ACTION_PUSH_VLAN: dr_dump_action_push_vlan,
    DR_DUMP_REC_TYPE_ACTION_METER: dr_dump_action_meter,
    DR_DUMP_REC_TYPE_ACTION_SAMPLER: dr_dump_action_sampler,
    DR_DUMP_REC_TYPE_ACTION_DEST_ARRAY: dr_dump_action_dest_array,
    DR_DUMP_REC_TYPE_ACTION_ASO_FIRST_HIT: dr_dump_action_aso_flow_hit,
    DR_DUMP_REC_TYPE_ACTION_ASO_FLOW_METER: dr_dump_action_aso_flow_meter,
    DR_DUMP_REC_TYPE_ACTION_MISS: dr_dump_action_default_miss,
    DR_DUMP_REC_TYPE_ACTION_ASO_CT: dr_dump_action_aso_ct,
    DR_DUMP_REC_TYPE_PMD_ACTION_PKT_REFORMAT: dr_dump_encap_decap,
    DR_DUMP_REC_TYPE_PMD_ACTION_COUNTER: dr_dump_counter,
    DR_DUMP_REC_TYPE_PMD_ACTION_MODIFY_HDR: dr_dump_modify_hdr,
    DR_DUMP_REC_TYPE_ACTION_ROOT_FT: dr_dump_action_root_ft,
    DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE: dr_dump_action_match_range,
}

unsupported_obj_list = []
def dr_report_unsupported_objects():
    for dr_rec_type in unsupported_obj_list:
        if dr_rec_type_is_domain(dr_rec_type):
            print("Err: Unsupported domain related object: ", dr_rec_type)
        elif dr_rec_type_is_table(dr_rec_type):
            print("Err: Unsupported table related object: ", dr_rec_type)
        elif dr_rec_type_is_matcher(dr_rec_type):
            print("Err: Unsupported matcher related object: ", dr_rec_type)
        elif dr_rec_type_is_rule(dr_rec_type):
            print("Err: Unsupported rule related object: ", dr_rec_type)
        elif dr_rec_type_is_action(dr_rec_type):
            if dump_ctx.rule:
                action = dr_dump_action_unsupported(line)
                dump_ctx.rule.add_action(action)
            else:
                print("Err: Unsupported domain related object: ", dr_rec_type)
        else:
            print("Unsupported object", dr_rec_type)

# parse csv record according to type and return parsed object (like dr_domain, dr_table, dr_rule ...)
def dr_csv_get_obj(line):
    rec_type = line[0]
    if rec_type not in switch_csv_rec_type.keys():
        return None

    parser = switch_csv_rec_type[line[0]]
    return parser(line)

def print_ctx(dump_ctx, view, verbose, raw, colored):
    dr_obj = None
    if dump_ctx.domain:
        dr_obj = dump_ctx.domain
    elif dump_ctx.table:
        dr_obj = dump_ctx.table.print_view
    elif dump_ctx.matcher:
        dr_obj = dump_ctx.matcher
    elif dump_ctx.rule:
        dr_obj = dump_ctx.rule

    if colored:
        set_colored_prints()

    if view == DR_DUMP_VIEW_TREE:
        dr_obj.print_tree_view(dump_ctx, verbose, raw)
    else:
        dr_obj.print_rule_view(dump_ctx, verbose, raw)

# Used to mark end of file
LAST_OBJ = dr_obj()

def parse_domain(csv_reader, domain_obj=None, verbose=0):
    """
    Function is parsing one domain from provided csv reader
    :param csv_reader: csv reader to parse with
    :param domain_obj: domain object that was parsed in previous function call
    :return: dump_ctx: parsed domain
              dr_obj: parsed domain object for the next function call (or
              LAST_OBJ when end of file is reached)
    """
    dump_ctx = dr_dump_ctx()

    if domain_obj:
        dump_ctx.domain = domain_obj

    for line in csv_reader:
        if len(line) <= 0:
            continue

        if line[0][0] == '4' and verbose < 4:
            continue

        dr_obj = dr_csv_get_obj(line)
        dr_rec_type = line[0]

        # report unsupported object
        if dr_obj is None:
            if dr_rec_type not in unsupported_obj_list:
                unsupported_obj_list.append(dr_rec_type)
            continue

        # update Rule entry objects
        if dr_rec_type in [DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V0,
                           DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V0,
                           DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V1,
                           DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V1]:

            definer_id = '-1'
            for builder in dump_ctx.matcher.builders[::-1]:
                if ((int(builder.data["is_rx"]) and dr_rec_type in [DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V0, \
                                                                    DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V1]) or \
                    ((not int(builder.data["is_rx"]) and dr_rec_type in [DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V0, \
                                                                         DR_DUMP_REC_TYPE_RULE_TX_ENTRY_V1]))):
                    definer_id = builder.data["definer_id"]

            dr_obj.data["definer_id"] = definer_id
            dump_ctx.rule.add_rule_entry(dr_obj)

        # update Action objects
        elif dr_rec_type_is_action(dr_rec_type):
            dump_ctx.rule.add_action(dr_obj)
            if dr_rec_type in [DR_DUMP_REC_TYPE_ACTION_ENCAP_L2,
                               DR_DUMP_REC_TYPE_ACTION_ENCAP_L3,
                               DR_DUMP_REC_TYPE_ACTION_CTR,
                               DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR]:
               dr_obj.add_dump_ctx(dump_ctx)

        # update Rule objects
        elif dr_rec_type == DR_DUMP_REC_TYPE_RULE:
            dump_ctx.rule = dr_obj
            dump_ctx.matcher.add_rule(dr_obj)

        # update Matcher objects
        elif dr_rec_type == DR_DUMP_REC_TYPE_MATCHER_BUILDER:
            dump_ctx.matcher.add_builder(dr_obj)

        elif dr_rec_type == DR_DUMP_REC_TYPE_MATCHER_RX or \
                dr_rec_type == DR_DUMP_REC_TYPE_MATCHER_TX:
            dump_ctx.matcher.add_matcher_rx_tx(dr_obj)

        elif dr_rec_type in [DR_DUMP_REC_TYPE_MATCHER_MASK,
                             DR_DUMP_REC_TYPE_MATCHER_MASK_WITH_RESERVED]:
            dump_ctx.matcher.add_mask(dr_obj)

        elif dr_rec_type == DR_DUMP_REC_TYPE_MATCHER:
            dump_ctx.matcher = dr_obj
            dump_ctx.table.add_matcher(dr_obj)

        # update Table objects
        elif dr_rec_type == DR_DUMP_REC_TYPE_TABLE_RX or \
                dr_rec_type == DR_DUMP_REC_TYPE_TABLE_TX:
            dump_ctx.table.add_table_rx_tx(dr_obj)

        elif dr_rec_type == DR_DUMP_REC_TYPE_TABLE:
            dump_ctx.table = dr_obj
            dump_ctx.domain.add_table(dr_obj)

        # update Domain objects
        elif dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN:
            # If parsing reached the next domain we return the parsed object to
            # use it for the next function call since we can't re-parse this
            # line again
            if dump_ctx.domain is not None:
                return dump_ctx, dr_obj
            dump_ctx.domain = dr_obj

        elif dump_ctx.domain and dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN_INFO_FLEX_PARSER:
            dump_ctx.domain.add_flex_parser(dr_obj)

        elif dump_ctx.domain and dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN_INFO_DEV_ATTR:
            dump_ctx.domain.add_dev_attr(dr_obj)

        elif dump_ctx.domain and dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT:
            dump_ctx.domain.add_vport(dr_obj)

        elif dump_ctx.domain and dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN_INFO_CAPS:
            dump_ctx.domain.add_caps(dr_obj)

        elif dump_ctx.domain and dr_rec_type == DR_DUMP_REC_TYPE_DOMAIN_SEND_RING:
            dump_ctx.domain.add_send_ring(dr_obj)

        elif dr_rec_type == DR_DUMP_REC_TYPE_PMD_ACTION_PKT_REFORMAT:
            if dr_obj.id and dr_obj.data:
                dump_ctx.encap_decap[dr_obj.id] = dr_obj.data

        elif dr_rec_type == DR_DUMP_REC_TYPE_PMD_ACTION_COUNTER:
            if dr_obj.id and dr_obj.data:
                dump_ctx.counter[dr_obj.id] = dr_obj.data

        elif dr_rec_type == DR_DUMP_REC_TYPE_PMD_ACTION_MODIFY_HDR:
            if dr_obj.id and dr_obj.data:
                dump_ctx.modify_hdr[dr_obj.id] = dr_obj.data

    return dump_ctx, LAST_OBJ

def parse_args():
    parser = argparse.ArgumentParser(description='''mlx_steering_dump.py - Steering dump tool''')
    parser.add_argument('-f', dest="FILEPATH", default="", help='input steering dump file path')
    parser.add_argument('-p', dest="dpdk_pid", type=int, default=-1,
                        help='Trigger DPDK app to generate CSV dump file (-p <APP PID>)')
    parser.add_argument('-t', action='store_true', default=False, dest='tree_view',
                        help='tree view (default is rule view)')
    parser.add_argument("-v", action="count", dest='verbose', default=0, help="increase output verbosity")
    parser.add_argument('-r', action='store_true', default=False, dest='raw', help='raw output')
    parser.add_argument('-c', action='store_true', default=False, dest='colored', help='colored output')
    parser.add_argument('-port', dest="dpdk_port", type=int, default=0,
                        help='Trigger DPDK app <PORT> (must provide PID with -p)')
    parser.add_argument('-flowptr', dest="flow_ptr", type=int, default=0, help='dump single rule by rte_flow_pointer')
    parser.add_argument('-version', action='store_true', default=False, dest='version', help='show version')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()
    if (args.version):
        print_dr(dr_print_color.RESET, "Version %s\n" % g_version)
        sys.exit(0)

    if (args.FILEPATH == ""):
        set_colored_prints()
        print_dr(dr_print_color.ERROR, "No input steering dump file provided (-f FILEPATH)\n")
        sys.exit(0)

    if (args.dpdk_pid > 0):
        if dr_trigger.trigger_dump(args.dpdk_pid, args.dpdk_port, args.FILEPATH, args.flow_ptr) is None:
            sys.exit(-1)
    domain_obj = None
    with open(args.FILEPATH) as csv_file:
        csv_reader = csv.reader(csv_file)
        while domain_obj != LAST_OBJ:
            dump_ctx, domain_obj = parse_domain(csv_reader, domain_obj, args.verbose)
            print_ctx(dump_ctx, DR_DUMP_VIEW_TREE if args.tree_view
            else DR_DUMP_VIEW_RULE, args.verbose,
                      args.raw, args.colored)
    if args.verbose:
        dr_report_unsupported_objects()

    if unsupported_obj_list:
        print_dr(dr_print_color.ERROR, "Warning: missing info due to unsupporteds objects\n")
    sys.exit(0)


class WordSequence():
    UNK_TAG = "<UNK>"
    PAD_TAG = "<PAD>"
    UNK = 1
    PAD = 0

    def __init__(self):
        self.dict = {
            self.UNK_TAG: self.UNK,
            self.PAD_TAG: self.PAD
        }
        self.count = {}
        self.inverse_dict = {}

    def fit(self, sentence):
        """
        统计词频
        :param sentence:
        :return:
        """
        for word in sentence:
            self.count[word] = self.count.get(word, 0) + 1

    def build_vocab(self, min_count=0, max_count=None, max_features=None):
        """
        根据条件构建 词典
        :param min_count:最小词频
        :param max_count: 最大词频
        :param max_features: 最大词语数
        :return:
        """
        if min_count is not None:
            self.count = {word: count for word, count in self.count.items() if count > min_count}
        if max_count is not None:
            self.count = {word: count for word, count in self.count.items() if count < max_count}
        if max_features is not None:
            # 排序
            self.count = dict(sorted(self.count.items(), lambda x: x[-1], reverse=True)[:max_features])

        for word in self.count:
            self.dict[word] = len(self.dict)  # 每次word对应一个数字

            # 把dict进行翻转
            self.inverse_dict = dict(zip(self.dict.values(), self.dict.keys()))

    def transform(self, sentence, max_len=None):
        """
        把句子转化为数字序列
        :param sentence:
        :return:
        """
        if len(sentence) > max_len:
            sentence = sentence[:max_len]
        else:
            sentence = sentence + [self.PAD_TAG] * (max_len - len(sentence))
        return [self.dict.get(i, 1) for i in sentence]

    def inverse_transform(self, incides):
        """
        把数字序列转化为字符
        :param incides:
        :return:
        """
        return [self.inverse_dict.get(i, "<UNK>") for i in incides]

    def __len__(self):
        return len(self.dict)

if __name__ == '__main__':
    sentences = [["今天", "天气", "很", "好"],
                 ["今天", "去", "吃", "什么"]]

    ws = WordSequence()
    for sentence in sentences:
        ws.fit(sentence)

    ws.build_vocab(min_count=0)
    print(ws.dict)
    ret = ws.transform(["今天", "好热", "呀", "呀", "呀", "呀", "呀", "呀", "呀"], max_len=5)
    print(ret)
    ret = ws.inverse_transform(ret)
    print(ret)


# Created by Andy at 03-Jan-21

# Enter description here

# ___________________________________

from abc import ABC, abstractmethod

class Validator(ABC):

    def __set_name__(self, owner, name):
        self.private_name = "_"+ name

    def __get__(self, obj, objtype = None):
        return getattr(obj, self.private_name)

    def __set__(self, obj, value):
        self.validate(value)
        return setattr(obj, self.private_name, value)

    @abstractmethod
    def validate(self, value):
        pass

class Number(Validator):
    def __init__(self, min_value = None, max_value = None, isint = False):
        self.min_value = min_value
        self.max_value = max_value
        self.is_int = isint

    def validate(self, value):
        if self.is_int and not isinstance(value, int):
            raise TypeError(f"Expected {value!r} to be an inteter")
        if not isinstance(value, (int,float)):
            raise TypeError(f"Expected {value!r} to be a float or an integer")
        if self.min_value is not None and value<self.min_value:
            raise ValueError(f"Expected value to be at least {self.min_value}")
        elif self.max_value is not None and value > self.max_value:
            raise ValueError(f"Expected {value} to be at most {self.max_value}")

class Settings():
    '''Settings object for the solver'''
    population_size = Number(1,1000, True)
    p_cross = Number(0,1)
    p_mutation = Number(0,1)

    def __init__(self, population_size=50, p_mutation=0.01, p_cross=0.5):
        self.population_size = int(population_size)
        self.p_mutation = p_mutation
        self.p_cross = p_cross
        self.mutator = None
        self.selector = None

if __name__ == "__main__":
    test_settings = Settings(population_size=50, p_mutation=-0.1, p_cross=-0.2)
    print(test_settings.p_cross)