# coding: utf-8

# # The UNSW-NB15 data set

# In[1]:

import pandas as pd

# In[2]:

unsw = pd.read_csv('UNSW_NB15_training-set.csv')

# In[3]:

unsw.columns

# In[4]:

cols_to_norm = ['sbytes','dbytes','sttl','dttl','sloss','dloss','spkts','sinpkt','dinpkt','smean','dmean','dpkts','trans_depth','sjit','djit','ct_flw_http_mthd','ct_ftp_cmd','ct_src_dport_ltm','ct_dst_sport_ltm','ct_dst_src_ltm']

# In[5]:

unsw[cols_to_norm] = unsw[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max()-x.min() ))

# In[6]:

import tensorflow as tf

# In[7]:

spkts = tf.feature_column.numeric_column('spkts')
dpkts = tf.feature_column.numeric_column('dpkts')
sbytes = tf.feature_column.numeric_column('sbytes')
dbytes = tf.feature_column.numeric_column('dbytes')
sttl = tf.feature_column.numeric_column('sttl')
dttl = tf.feature_column.numeric_column('dttl')
sloss = tf.feature_column.numeric_column('sloss')
dloss = tf.feature_column.numeric_column('dloss')
sinpkt = tf.feature_column.numeric_column('sinpkt')
dinpkt = tf.feature_column.numeric_column('dinpkt')
sjit = tf.feature_column.numeric_column('sjit')
djit = tf.feature_column.numeric_column('djit')
smean = tf.feature_column.numeric_column('smean')
dmean = tf.feature_column.numeric_column('dmean')
trans_depth = tf.feature_column.numeric_column('trans_depth')
ct_flw_http_mthd = tf.feature_column.numeric_column('ct_flw_http_mthd')
ct_ftp_cmd = tf.feature_column.numeric_column('ct_ftp_cmd')
ct_src_dport_ltm = tf.feature_column.numeric_column('ct_src_dport_ltm')
ct_dst_sport_ltm = tf.feature_column.numeric_column('ct_dst_sport_ltm')
ct_dst_src_ltm = tf.feature_column.numeric_column('ct_dst_src_ltm')

# In[8]:

assigned_grp = tf.feature_column.categorical_column_with_vocabulary_list('attack_cat', ['Fuzzers', 'Analysis', 'Backdoors', 'DoS', 'Exploits', 'Generic', 'Reconnaissance', 'Shellcode', 'Worms'])

# In[9]:

import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

# In[10]:

feat_cols = [spkts,dpkts, sbytes, dbytes, sttl, dttl, sloss, dloss, sinpkt,dinpkt, smean, dmean, sjit, djit, trans_depth, ct_ftp_cmd, ct_src_dport_ltm, ct_dst_sport_ltm, ct_dst_src_ltm, ct_flw_http_mthd]

# In[11]:

x_data = unsw.drop('label', axis = 1)

# In[12]:

labels = unsw['label']

# In[13]:

from sklearn.model_selection import train_test_split

# In[14]:

X_train, x_test, y_train, y_test = train_test_split(x_data, labels, test_size = 0.3, random_state = 101)

# In[15]:

input_func = tf.estimator.inputs.pandas_input_fn(x = X_train, y = y_train, batch_size=10, num_epochs=1000, shuffle=True)

# In[16]:

model = tf.estimator.LinearClassifier(feature_columns=feat_cols,n_classes=9,)

# In[17]:

model.train(input_fn=input_func, steps=1000)

# In[18]:

unsw.columns

# In[19]:

x_data.columns

# In[20]:

x_data = x_data.drop('state', axis = 1)

# In[21]:

x_data.head()

# In[22]:

x_data.columns

# In[23]:

cols_to_norm = ['dur', 'spkts', 'dpkts', 'sbytes', 'dbytes', 'rate',
       'sttl', 'dttl', 'sload', 'dload', 'sloss', 'dloss', 'sinpkt', 'dinpkt',
       'sjit', 'djit', 'swin', 'stcpb', 'dtcpb', 'dwin', 'synack', 'ackdat',
       'smean', 'dmean']

# In[24]:

x_data[cols_to_norm] = x_data[cols_to_norm].apply(lambda x: (x - x.min())/(x.max()-x.min()))

# In[25]:

eval_input_function = tf.estimator.inputs.pandas_input_fn(x = x_test, y = y_test, batch_size=10, num_epochs=1,shuffle=False)

# In[26]:

results = model.evaluate(eval_input_function)

# In[27]:

results

# In[28]:

pred_input_func = tf.estimator.inputs.pandas_input_fn(x = x_test, batch_size=10, num_epochs=1, shuffle=False)

# In[29]:

predictions = model.predict(pred_input_func)

# In[30]:

my_pred = list(predictions)

# In[31]:

my_pred

# ## DNN classifier

# In[32]:

embedded_group_col = tf.feature_column.embedding_column(assigned_grp, dimension=2)

# In[33]:

feat_cols = [spkts,dpkts, sbytes, dbytes, sttl, dttl, sloss, dloss, sinpkt,dinpkt, smean, dmean, sjit, djit, trans_depth, ct_ftp_cmd, ct_src_dport_ltm, ct_dst_sport_ltm, ct_dst_src_ltm, ct_flw_http_mthd, embedded_group_col]

# In[34]:

input_func = tf.estimator.inputs.pandas_input_fn(X_train, y_train, batch_size=10, num_epochs=1000, shuffle=True)

# In[35]:

dnn_model = tf.estimator.DNNClassifier(hidden_units=[3,5,3], feature_columns=feat_cols, n_classes=2)

# In[36]:

dnn_model.train(input_fn=input_func, steps = 1000)

# In[37]:

eval_input_func = tf.estimator.inputs.pandas_input_fn(x = x_test, y = y_test, num_epochs=1, batch_size=10, shuffle=False)

# In[38]:

dnn_model.evaluate(eval_input_func)


'''Magic Square

https://en.wikipedia.org/wiki/Magic_square

A magic square is a n * n square grid filled with distinct positive integers in
the range 1, 2, ..., n^2 such that each cell contains a different integer and
the sum of the integers in each row, column, and diagonal is equal.

'''

from z3 import *

def solve_magic_square(n, r, c, val):
    solver = Solver()

    ### CREATE CONSTRAINTS AND LOAD STORE THEM IN THE SOLVER
    # nxn matrix of integer variables
    X = [[Int("x_%s_%s" % (i, j)) for j in range(n)]
         for i in range(n)]

    # each cell contains a value in {1, ..., n}
    cells_c = [And(1 <= X[i][j], X[i][j] <= pow(n, 2))
               for i in range(n) for j in range(n)]

    # each cell contains a dierent integer
    sq_c = [Distinct([X[i][j]
                      for i in range(n) for j in range(n)])]

    # sum
    sum_rows_c = [Sum(X[i_1]) == Sum(X[i_2]) for i_1 in range(n) for i_2 in range(n)]
    sum_cols_c = [Sum([X[i][j] for i in range(n)]) == Sum(X[0])
                  for j in range(n)]

    sum_diagonal_c1 = [Sum([X[i][i] for i in range(n)]) == Sum(X[0])]
    sum_diagonal_c2 = [Sum([X[i][n - i - 1] for i in range(n)]) == Sum(X[0])]
    # set val to X(r,c)
    set_c = [X[r][c] == val]

    magic_square_c = cells_c + sq_c + set_c + sum_rows_c + sum_cols_c \
                     + sum_diagonal_c1 + sum_diagonal_c2

    solver.add(magic_square_c)

    if solver.check() == sat:

        mod = solver.model()

        ### CREATE RESULT MAGIC SQUARE BASED ON THE MODEL FROM THE SOLVER
        res = [[mod.evaluate(X[i][j]).as_long() for j in range(n)]
               for i in range(n)]
        return res
    else:
        return None

def print_square(square):
    '''
    Prints a magic square as a square on the console
    '''
    n = len(square)

    assert n > 0
    for i in range(n):
        assert len(square[i]) == n

    for i in range(n):
        line = []
        for j in range(n):
            line.append(str(square[i][j]))
        print('\t'.join(line))

def puzzle(n, r, c, val):
    res = solve_magic_square(n, r, c, val)
    if res is None:
        print('No solution!')
    else:
        print('Solution:')
        print_square(res)

if __name__ == '__main__':
    n = 3
    r = 0
    c = 0
    val = 8
    puzzle(n, r, c, val)


import torch
import string
import os
from torch.utils.data import Dataset
import cv2

class CaptchasDataset(Dataset):
    def __init__(
        self, images_path, ground_truth, transpose=True, label_as_str=False
    ):
        self.ground_truth = ground_truth
        self.images_path = images_path
        self.transpose = transpose
        self.label_as_str = label_as_str
        self.label_to_index = {
            char: i for i, char in enumerate(string.printable)
        }
        self.index_to_label = {
            i: char for i, char in enumerate(string.printable)
        }

    def __len__(self):
        return len(self.ground_truth)

    def __getitem__(self, index):
        if isinstance(index, slice):
            return [self[i] for i in range(index.start, index.stop)]
        elif torch.is_tensor(index):
            index = index.tolist()

        item = self.ground_truth.loc[index]
        image_path = os.path.join(self.images_path, item[0])
        image = cv2.imread(image_path)
        # OpenCV reads by default as BGR.
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transpose:
            # 200x50x3 => 3x200x50
            image = image.transpose((2, 0, 1))
        if self.label_as_str:
            return image, item[1]
        return image, [self.label_to_index[x] for x in item[1]]


from django.contrib import admin
from .models import LaraEnquiry, LaraEnquiryForm, FollowUp

class LaraAdmin(admin.ModelAdmin):
    list_filter = ('date',)
    fields = (
        'name',
        'mobile',
        'email',
        'pass_out_Year',
        'latest_education',
        'branch',
        'percentages',
        'university_or_college',
        'state',
        'appointment_date',
        'Gender',
    )
    form = LaraEnquiryForm
    list_display = ('name', 'mobile', 'email', 'appointment_date')
    # search_fields = ('name', 'mobile')

admin.site.register(LaraEnquiry, LaraAdmin)
admin.site.register(FollowUp)


from pydantic import BaseModel, validator
from typing import Dict, List, Optional
from paprika.setup import _get_installed_benchmarks
from paprika.restraints.read_yaml import read_yaml
from pint import UnitRegistry
from pint.quantity import _Quantity
from pathlib import Path

ureg = UnitRegistry()

def check_atoms_are_string(atoms):
    if isinstance(atoms, str):
        return
    else:
        raise ValueError(f"Restraint atom definition not detected as `str`: {atoms}")

def check_force_constant_is_float(force_constant):
    if isinstance(force_constant, float):
        return
    else:
        raise ValueError(f"Restraint force constant not detected as `float`: {force_constant}")

def check_restraint_target_is_float(restraint_target):
    if isinstance(restraint_target, float):
        return
    else:
        raise ValueError(f"Restraint target value not detected as `float`: {restraint_target}")

def check_restraint_by_phase(restraint, phase):
    if "force_constant" not in restraint[phase].keys():
        raise ValueError(f"Restraints missing atom force constant: {restraint}")
    check_force_constant_is_float(restraint[phase]["force_constant"])
    check_restraint_target_is_float(restraint[phase]["target"])

class TaproomHost(BaseModel):
    name: str
    path: Path
    structure: str
    net_charge: float
    resname: str
    aliases: Optional[List[Dict]]
    restraints: List[Dict]
    attach_windows: int
    pull_windows: int
    release_windows: int
    lambda_attach: List[float]
    lambda_release: List[float]
    pull_value: float
    waters: int
    water_model: str

    class Config:
        arbitrary_types_allowed = True

    @validator("structure")
    def structure_exists(cls, value, values):
        if not Path.exists(values["path"].parent.joinpath(value)):
            raise ValueError(f"Structure file appears not to exist: {values['path'].parent.joinpath(value)}")
        return value

    @validator("restraints")
    def restraints_have_atoms_and_values(cls, value):
        if not value["atoms"]:
            raise ValueError(f"Restraints missing atom definition: {value}")
        if not value["force_constant"]:
            raise ValueError(f"Restraints missing atom force constant: {value}")
        if not isinstance(value["atoms"], str):
            raise ValueError(f"Restraint atom definition not detected as `str`: {value}")
        if not isinstance(value["force_constant"], float):
            raise ValueError(f"Restraint force constant definition not detected as `float`: {value}")
        return value

class TaproomGuest(BaseModel):
    name: str
    path: Path
    structure: str
    complex: str
    net_charge: float
    aliases: Optional[List[Dict]]
    restraints: List[Dict]
    microstates: int

    class Config:
        arbitrary_types_allowed = True

    @validator("structure")
    def structure_exists(cls, value, values):
        if not Path.exists(values["path"].parent.joinpath(value)):
            raise ValueError(f"Structure file appears not to exist: {values['path'].parent.joinpath(value)}")
        return value

    @validator("complex")
    def at_least_one_complex_exists(cls, value, values):
        complex_prefix = Path(value).stem
        p = list(values["path"].parent.glob(f"{complex_prefix}*.pdb"))
        if len(p) < 1:
            raise ValueError(f"Did not find at least one complex file: {values['path'].parent.joinpath(f'{complex_prefix}*.pdb')}")
        return value

    @validator("restraints")
    def restraints_have_atoms_and_values(cls, value):
        if not value["atoms"]:
            raise ValueError(f"Restraints missing atom definition: {value}")

        keys = value.keys()

        if "attach" in keys:
            check_restraint_by_phase(value, "attach")
        if "pull" in keys:
            check_restraint_by_phase(value, "pull")
        if "release" in keys:
            check_restraint_by_phase(value, "release")

        if "force_constant" in keys():
            check_force_constant_is_float(value["foce_constant"])
        if "target" in keys():
            check_restraint_target_is_float(value["target"])

        return value

class TaproomMeasurement(BaseModel):
    temperature: _Quantity
    pressure: _Quantity
    pH: float = 6.90
    name: str = "guest"
    SMILES: str = "C"
    buffer: List[Dict]
    technique: str = "ITC"
    delta_G: _Quantity
    delta_G_uncertainty: _Quantity
    delta_H: _Quantity
    delta_H_uncertainty: _Quantity
    stoichiometry: str = "1:1"
    comment: str = ""
    doi: str = ""
    unusual: bool = False

    class Config:
        arbitrary_types_allowed = True

    @validator("temperature")
    def temperature_is_positive(cls, value):
        if value.magnitude < 0:
            raise ValueError(f"Temperature must be positive: {value}")
        if not value.check('[temperature]'):
            raise ValueError(f"Temperature in correct units: {value}")
        return value

    @validator("delta_G", "delta_G_uncertainty", "delta_H", "delta_H_uncertainty")
    def measurement_units(cls, value):
        if not value.check('[length] ** 2 * [mass] / [substance] / [time] ** 2'):
            raise ValueError(f"Experimental value is not in correct units: {value}")
        return value

installed_benchmarks = _get_installed_benchmarks()
yaml_data = read_yaml(
    installed_benchmarks["host_guest_measurements"]["acd"]["bam"]["yaml"]
)

tmp = TaproomMeasurement(
    temperature=ureg(yaml_data["state"]["temperature"]),
    pressure=ureg(yaml_data["state"]["pressure"]),
    pH=yaml_data["state"]["pH"],
    name=yaml_data["substance"]["name"],
    SMILES=yaml_data["substance"]["SMILES"],
    buffer=yaml_data["substance"]["buffer"],
    technique=yaml_data["measurement"]["technique"],
    delta_G=ureg(yaml_data["measurement"]["delta_G"]),
    delta_G_uncertainty=ureg(yaml_data["measurement"]["delta_G_uncertainty"]),
    delta_H=ureg(yaml_data["measurement"]["delta_H"]),
    delta_H_uncertainty=ureg(yaml_data["measurement"]["delta_H_uncertainty"]),
    comment=yaml_data["measurement"]["comment"],
    doi=yaml_data["provenance"]["doi"],
    unusual=yaml_data["unusual"]
)

yaml_data = read_yaml(
    installed_benchmarks["host_guest_systems"]["acd"]["yaml"]["p"]
)

all_restraints = []
for restraint in yaml_data["restraints"]["static"]:
    all_restraints.append(restraint)
for restraint in yaml_data["restraints"]["conformational"]:
    all_restraints.append(restraint)

tmp = TaproomHost(
    name=yaml_data["name"],
    path=installed_benchmarks["host_guest_systems"]["acd"]["yaml"]["p"],
    structure=yaml_data["structure"],
    net_charge=yaml_data["net_charge"],
    resname=yaml_data["resname"],
    aliases=yaml_data["aliases"],
    restraints=all_restraints,
    attach_windows=yaml_data["calculation"]["windows"]["attach"],
    pull_windows=yaml_data["calculation"]["windows"]["pull"],
    release_windows=yaml_data["calculation"]["windows"]["release"],
    lambda_attach=yaml_data["calculation"]["lambda"]["attach"],
    lambda_release=yaml_data["calculation"]["lambda"]["release"],
    pull_value=yaml_data["calculation"]["target"]["pull"],
    waters=yaml_data["calculation"]["system"]["waters"],
    water_model=yaml_data["calculation"]["system"]["model"]
)

yaml_data = read_yaml(
    installed_benchmarks["host_guest_systems"]["acd"]["bam"]["yaml"]
)

all_restraints = []
restraint_types = yaml_data["restraints"].keys()

for restraint_type in restraint_types:
    for restraint in yaml_data["restraints"][restraint_type]:
        all_restraints.append(restraint)

tmp = TaproomGuest(
    name=yaml_data["name"],
    path=installed_benchmarks["host_guest_systems"]["acd"]["bam"]["yaml"],
    structure=yaml_data["structure"],
    complex=yaml_data["complex"],
    net_charge=yaml_data["net_charge"],
    aliases=yaml_data["aliases"],
    restraints=all_restraints,
    microstates=yaml_data["symmetry_correction"]["microstates"]
)


# Copyright 2018 Platform9 Systems, Inc.

# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at

#   http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import openstack
from keystoneauth1.exceptions import MissingRequiredOptions

# Local imports
from QbertClient import QbertClient
from Kubeconfig import Kubeconfig

LOG = logging.getLogger(__name__)

class Fetcher(object):
    """ Fetches a Kubeconfig from Qbert """
    def __init__(self, kubeconfig=None, os_config=None, os_cloud=None):
        """ From a defined OpenStack Cloud, initialize QbertClient

        If cloud_name is not specified, the default values 'envvars' and 'defaults' will be used.
        The cloud 'envvars' will be preferred to 'defaults' if found.

        Args:
            kubeconfig: qbertconfig.Kubeconfig object
            os_config: an openstack.config.cloud_config.CloudConfig object
            os_cloud: an openstack.config.cloud_region.CloudRegion object
        Returns:
            An initialized qbertconfig.QbertClient object
        """
        self.kubeconfig = kubeconfig
        self.os_config = os_config
        self.os_cloud = os_cloud

        if not self.kubeconfig:
            self.kubeconfig = Kubeconfig()

        self.qbert_session = self._initialize_qbert_client(os_config, os_cloud)

    def _initialize_qbert_client(self, os_config=None, os_cloud=None):
        """ From a defined OpenStack Cloud, initialize QbertClient

        If cloud_name is not specified, the default values 'envvars' and 'defaults' will be used.
        The cloud 'envvars' will be preferred to 'defaults' if found.

        Args:
            os_config: an openstack.config.cloud_config.CloudConfig object

        Returns;
            An initialized qbertconfig.QbertClient object
        """
        if os_cloud:
            self.os_cloud = os_cloud
            return QbertClient(os_cloud=self.os_cloud)

        # Cloud not provided, so see if we need to create config and then get cloud
        if not os_config:
            # this will decide whether to use env vars, or a clouds.yaml
            self.os_config = openstack.config.OpenStackConfig()

        # Now use config to get a cloud
        try:
            self.os_cloud = self.os_config.get_one_cloud()
        except MissingRequiredOptions as e:
            # If this fails, it means no other credentials were provided another way
            LOG.error("Unable to validate openstack credentials. Check this error out: \n" + e.message)
            LOG.error("Check to ensure your OpenStack credentials are in clouds.yaml"
                      " or available as environment variables")
            raise e
        return QbertClient(os_cloud=self.os_cloud)

    def save(self):
        """ Saves the current kubeconfig to file """
        self.kubeconfig.save()

    def fetch(self, cluster_name=None, cluster_uuid=None):
        """
        Using the qbert API, download a kubeconfig file for the specified cluster

        Args:
            cluster_name: name of the qbert cluster
            cluster_uuid: ID of the qbert cluster

        Returns:
            KubeConfig object
        """
        LOG.debug("Cluster: '%s' (%s)", cluster_name, cluster_uuid)
        cluster = self.qbert_session.find_cluster(cluster_uuid, cluster_name)
        new_kubeconfig = Kubeconfig(kcfg=self.qbert_session.get_kubeconfig(cluster))
        self.kubeconfig.merge_kubeconfigs(new_kubeconfig)
        return self.kubeconfig


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
   File Name：     gen_match_feature
   Description :
   Author :       Administrator
   date：          2018/6/18 0018
-------------------------------------------------
   Change Activity:
                   2018/6/18 0018:
-------------------------------------------------
"""
__author__ = 'Administrator'

import pandas as pd
import numpy as np
import nltk
from nltk.corpus import stopwords

nltk.download('stopwords')
nltk.download('punkt')

stop_word = ['L1187','L1368','L0362','L1128','LL32','L0143','L3019','L2218','L2582','L1861'
             ,'L2214','L0104']

question = pd.read_csv('../data/question.csv')
question = question[['qid','chars']]

train = pd.read_csv('../data/train.csv')
test = pd.read_csv('../data/test.csv')
train = pd.merge(train,question,left_on=['q1'],right_on=['qid'],how='left')
train = pd.merge(train,question,left_on=['q2'],right_on=['qid'],how='left')
train = train[['label','chars_x','chars_y']]
train.columns = ['label','question1','question2']
# len_train = train.shape[0]
len_train = len(train)

test = pd.merge(test,question,left_on=['q1'],right_on=['qid'],how='left')
test = pd.merge(test,question,left_on=['q2'],right_on=['qid'],how='left')
test = test[['chars_x','chars_y']]
test.columns = ['question1','question2']

df_feat = pd.DataFrame()
df_data = pd.concat([train,test], axis=0)

# stops = set(stopwords.words(stop_word))
stops = stop_word
def word_match_share(row):
    q1words = {}
    q2words = {}
    for word in str(row['question1']).lower().split():
        if word not in stops:
            q1words[word] = 1
    for word in str(row['question2']).lower().split():
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        return 0
    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))
    return R

df_feat['word_match_ratio'] = df_data.apply(word_match_share, axis=1)

# Unigrams without stops
def get_unigrams(que):
    return [word for word in nltk.word_tokenize(que.lower()) if word not in stops]
df_data["unigrams_nostop_q1"] = df_data['question1'].apply(lambda x: get_unigrams(str(x)))
df_data["unigrams_nostop_q2"] = df_data['question2'].apply(lambda x: get_unigrams(str(x)))

def get_word_match_nostop_unigrams_count(row):
    return len(set(row["unigrams_nostop_q1"]).intersection(set(row["unigrams_nostop_q2"])))

def get_word_match_nostop_unigrams_ratio(row):
    return float(row["unigrams_nostop_match_count"])/max(len(set(row["unigrams_nostop_q1"]).union(set(row["unigrams_nostop_q2"]))),1)

unigrams_nostop_match_count = []
unigrams_nostop_match_ratio = []

for index, row in df_data.iterrows():
    count = len(set(row["unigrams_nostop_q1"]).intersection(set(row["unigrams_nostop_q2"])))
    ratio = float(count) / max(len(set(row["unigrams_nostop_q1"]).union(set(row["unigrams_nostop_q2"]))),1)
    unigrams_nostop_match_count.append(count)
    unigrams_nostop_match_ratio.append(ratio)

df_feat["unigrams_nostop_match_count"] = unigrams_nostop_match_count
df_feat["unigrams_nostop_match_ratio"] = unigrams_nostop_match_ratio

# Bigrams without stops
# ngrams方法
def get_bigrams(que):
    return [i for i in nltk.ngrams(que, 2)]
df_data["bigrams_nostop_q1"] = df_data["unigrams_nostop_q1"].apply(lambda x: get_bigrams(x))
df_data["bigrams_nostop_q2"] = df_data["unigrams_nostop_q2"].apply(lambda x: get_bigrams(x))

bigrams_nostop_match_count = []
bigrams_nostop_match_ratio = []

for index, row in df_data.iterrows():
    count = len(set(row["bigrams_nostop_q1"]).intersection(set(row["bigrams_nostop_q2"])))
    ratio = float(count) / max(len(set(row["bigrams_nostop_q1"]).union(set(row["bigrams_nostop_q2"]))),1)
    bigrams_nostop_match_count.append(count)
    bigrams_nostop_match_ratio.append(ratio)

df_feat["bigrams_nostop_match_count"] = bigrams_nostop_match_count
df_feat["bigrams_nostop_match_ratio"] = bigrams_nostop_match_ratio

# Trigrams without stops
def get_trigrams(que):
    return [i for i in nltk.ngrams(que, 3)]
df_data["trigrams_nostop_q1"] = df_data["unigrams_nostop_q1"].apply(lambda x: get_trigrams(x))
df_data["trigrams_nostop_q2"] = df_data["unigrams_nostop_q2"].apply(lambda x: get_trigrams(x))

trigrams_nostop_match_count = []
trigrams_nostop_match_ratio = []

for index, row in df_data.iterrows():
    count = len(set(row["trigrams_nostop_q1"]).intersection(set(row["trigrams_nostop_q2"])))
    ratio = float(count) / max(len(set(row["trigrams_nostop_q1"]).union(set(row["trigrams_nostop_q2"]))),1)
    trigrams_nostop_match_count.append(count)
    trigrams_nostop_match_ratio.append(ratio)

df_feat["trigrams_nostop_match_count"] = trigrams_nostop_match_count
df_feat["trigrams_nostop_match_ratio"] = trigrams_nostop_match_ratio

path = '../data/pre_feature/'
df_feat[:len_train].to_csv(path + 'train_feature_match.csv', index=False)
df_feat[len_train:].to_csv(path + 'test_feature_match.csv', index=False)


# Define a procedure, median, that takes three
# numbers as its inputs, and returns the median
# of the three numbers.

# Make sure your procedure has a return statement.

def bigger(a,b):
    if a > b:
        return a
    else:
        return b

def biggest(a,b,c):
    return bigger(a,bigger(b,c))

def median(x,y,z):
    largest = biggest(x,y,z)
    if x == largest:
        return bigger(y,z)
    elif y == largest:
        return bigger(x,z)
    else:
        return bigger(x,y)

print(median(1,2,3))
#>>> 2

print(median(9,3,6))
#>>> 6

print(median(7,8,7))
#>>> 7

print(median(3,2,1))
#>>> 2


#! /usr/bin/env python

import sys
import numpy as np

NTIMES = 2048
NTIME_PER_PKT = 2
NCHAN_PER_PKT = 384
NANT_PER_PKT = 3
NANT_TOTAL = 192
NPOL_PER_PKT = 2

with open(sys.argv[1], 'r') as fh:
    raw = fh.read()

x = np.fromstring(raw, dtype='B')

x = x.reshape(NTIMES/NTIME_PER_PKT, NANT_TOTAL, NCHAN_PER_PKT, NTIME_PER_PKT, NPOL_PER_PKT)

tvg = np.zeros([NTIMES/NTIME_PER_PKT, NANT_TOTAL, NCHAN_PER_PKT, NTIME_PER_PKT, NPOL_PER_PKT])

ramp = np.arange(384) % 256
for t in range(NTIMES/NTIME_PER_PKT):
    for t_pkt in range(NTIME_PER_PKT):
        for a in range(NANT_TOTAL):
            for p in range(NPOL_PER_PKT):
                tvg[t, a, :, t_pkt, p] = (np.arange(384) + 2*(a%3) + p) % 256

for a in range(NANT_TOTAL):
    print("Antenna %d: OK?:" % a, end=' ')
    print(np.all(tvg[:,a,:,:,:] == x[:,a,:,:,:]))


import tensorflow as tf

from data import get_batch, get_full

class BasicAutoEncoder:
    """A basic autoencoder with a single hidden layer"""

    def __init__(self, data_x, data_x_, hidden_dim, epoch=1000, batch_size=50):
        self.data_x_ = data_x_
        self.data_x = data_x
        self.batch_size = batch_size
        self.epoch = epoch
        self.hidden_dim = hidden_dim
        self.input_dim = len(data_x[0])
        self.hidden_feature = []
        # print(data_x.shape, data_x_.shape, hidden_dim)

    def forward(self, x):
        with tf.name_scope('encode'):
            weights = tf.Variable(tf.random_normal([self.input_dim, self.hidden_dim], dtype=tf.float32),
                                  name='weights')

            biases = tf.Variable(tf.zeros([self.hidden_dim]), name='biases')
            encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases, name='encoded')

        with tf.name_scope('decode'):
            biases = tf.Variable(tf.zeros([self.input_dim]), name='biases')
            decoded = tf.matmul(encoded, tf.transpose(weights)) + biases
        # loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(decoded, x, name='cross_entropy'))
        return encoded, decoded

    def train(self, x_, decoded):
        loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(x_, decoded))))
        train_op = tf.train.AdamOptimizer(0.001).minimize(loss)
        return loss, train_op

    def run(self):
        with tf.Graph().as_default():
            x = tf.placeholder(dtype=tf.float32, shape=[None, self.input_dim], name='x')
            x_ = tf.placeholder(dtype=tf.float32, shape=[None, self.input_dim], name='x_')
            encoded, decoded = self.forward(x)
            loss, train_op = self.train(x_, decoded)
            with tf.Session() as sess:
                sess.run(tf.initialize_all_variables())
                for i in range(self.epoch):
                    for j in range(50):
                        b_x, b_x_ = get_batch(self.data_x, self.data_x_, self.batch_size)
                        sess.run(train_op, feed_dict={x: b_x, x_: b_x_})
                    if i % 100 == 0:
                        l = sess.run(loss, feed_dict={x: self.data_x, x_: self.data_x_})
                        print('epoch {0}: global loss = {1}'.format(i, l))
                self.hidden_feature = sess.run(encoded, feed_dict={x: self.data_x_})
                # print(sess.run(decoded, feed_dict={x: self.data_x})[0])

    def get_hidden_feature(self):
        return self.hidden_feature


# -*- coding: utf-8 -*-
import os
import glob
import re
import time
import numpy as np
import pandas as pd
import csv
from tqdm import tqdm
from colorama import Fore, Style
import nltk
from nltk.stem.snowball import SnowballStemmer
from nltk.corpus import stopwords
from nltk.tokenize import TweetTokenizer
from nltk.sentiment import SentimentIntensityAnalyzer
from google_trans_new import google_translator

def clean_text(text, remove_contractions = False, twitter = False):
    regrex_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\U00002500-\U00002BEF"  # chinese char
        u"\U00002702-\U000027B0"
        u"\U00002702-\U000027B0"
        u"\U000024C2-\U0001F251"
        u"\U0001f926-\U0001f937"
        u"\U00010000-\U0010ffff"
        u"\u2640-\u2642"
        u"\u2600-\u2B55"
        u"\u200d"
        u"\u23cf"
        u"\u23e9"
        u"\u231a"
        u"\ufe0f"
        u"\u3030"
        "]*", flags=re.UNICODE)
    text = re.sub(regrex_pattern,'',text)
    # removing & character that appers during encoding
    text = re.sub(r'&', '', str(text))
    # removing urls, in twitter all begins with https:t.co
    if twitter: text = re.sub(r"https?:\/\/[t.co]+\/*[\d|\w]*", '', str(text))

    if remove_contractions:
        text = re.sub(r"won\'t", "will not", text)
        text = re.sub(r"can\'t", "can not", text)
        text = re.sub(r"can\'t", "can not", text)
        text = re.sub(r"n\'t", " not", text)
        text = re.sub(r"\'re", " are", text)
        text = re.sub(r"\'s", " is", text)
        text = re.sub(r"\'d", " would", text)
        text = re.sub(r"\'ll", " will", text)
        text = re.sub(r"\'t", " not", text)
        text = re.sub(r"\'ve", " have", text)
        text = re.sub(r"\'m", " am", text)
    return text

def preprocessing_text(text, language, twitter = False):
    try:
        stops = set(stopwords.words(language))
        stemming = SnowballStemmer(language)
        pass
    except (OSError, ValueError):
        stops = set(stopwords.words('english'))
        stemming = SnowballStemmer('english')
        pass
    # clear text from emoticons, chinese chars, urls and so on
    text = clean_text(text, language == 'english', twitter)
    # tokenization
    if twitter:
        question = '?' in text.split('#')[0]
        tk = TweetTokenizer()
        tokens = tk.tokenize(text)
    else: tokens = nltk.word_tokenize(text)
    # all words in lower case and remove stop words and special chars
    tokens = [w.lower() for w in tokens if ((w.isalnum() and not twitter) or (w.isalnum() or '#'in w or '@' in w)) and not w in stops]
    # stemming
    tokens = [stemming.stem(w) for w in tokens]
    return [tokens, question] if twitter else tokens

def standardizing_date(date):
    month_to_number = {'january': '01', 'february': '02', 'march': '03', 'april': '04', 'may': '05', 'june': '06',
                       'july': '07', 'august': '08', 'september': '09', 'october': '10', 'november': '11', 'december': '12'}
    date = date.split(' ')
    date[0] = month_to_number[date[0].lower()]
    if len(date[1]) == 1: date[1] = '0'+date[1]
    resulting_date = date[2]+date[0]+date[1]
    return resulting_date

def computing_score(score):
    score = score.split(' ')[0]
    if score == 'no': return None
    else: return int(float(score))

def verified_purchase_boolean(ver_pur):
    if ver_pur.lower() == 'verified purchase':
        return True
    else:
        return False

def detect_language_from_text(text, title = None):
    text_wout_hashtag = text.split('#')[0]
    if len(text_wout_hashtag) > 0: text = text_wout_hashtag
    if text.isspace() or text == None:
        if title:
            text = title
        else:
            return 'english'
    try:
        text = text[:3000]
        pass
    except Exception:
        return 'english'
    for attempt in range(10):
        try:
            detector = google_translator()
            lang = detector.detect(text)[1]
            return lang
        except Exception as e:
            print(Style.BRIGHT+Fore.RED + '\n  Error: '+ str(e)+ Style.RESET_ALL)
            time.sleep(10)
            continue
        break
    return 'error'

def detecting_upvotes(upvotes):
    upvotes = upvotes.split(' ')[0]
    upvotes = re.sub(",", "", upvotes)
    if upvotes.lower() == 'one':
        return 1
    return int(upvotes)

def is_positive(sia, text: str) -> int:
    """True if text has positive compound sentiment, False otherwise."""
    return 1 if sia.polarity_scores(text)["compound"] > 0 else -1

def cleaning_amazon_datasets():
    os.chdir("G:\Il mio Drive\Project_Data_Mining\Data\Amazon")
    datasets = [f for f in glob.glob("*.csv")]
    for name_dataset in datasets:
        print("Checking operations on dataset:'"+name_dataset+"'")
        if os.path.exists('./clean_dataset/clean_'+name_dataset):
            cleaning = False
        else:
            cleaning = True
        if os.path.exists('./preprocessed_dataset/preprocessed_'+name_dataset) and not cleaning:
            preprocessing = False
            print('  > Dataset already cleaned and preprocessed')
        else:
            preprocessing = True
        time.sleep(1)

        if cleaning:
            with open(name_dataset, encoding='utf-8-sig', errors='replace') as f:
                df = pd.read_csv(f, sep=',', names=['name','score','title','place','date','verified purch','text','upvotes'])
                name = df['name'].to_dict()
                score = df['score'].to_dict()
                title = df['title'].to_dict()
                place = df['place'].to_dict()
                date = df['date'].to_dict()
                ver_purchase = df['verified purch'].to_dict()
                text = df['text'].to_dict()
                upvotes = df['upvotes'].to_dict()
            new_df = {}
            for i in tqdm(name, desc='Cleaning dataset'):
                if i > 0:
                    new_score = computing_score(score[i])
                    new_date = standardizing_date(date[i])
                    new_ver_pur = verified_purchase_boolean(ver_purchase[i])
                    new_upvotes = detecting_upvotes(upvotes[i])
                    if title[i] == 'title not found': title[i] = None
                    language = detect_language_from_text(text[i], title[i])
                    new_df[i] = [name[i], new_score, title[i], place[i], new_date, new_ver_pur, text[i], new_upvotes, language]
                    time.sleep(1)

            with open('./clean_dataset/clean_'+name_dataset, 'wt', encoding='utf-8', newline='') as of:
                writer = csv.writer(of, delimiter = ',')
                writer.writerow(['name','score','title','place','date','verified purch','text','upvotes', 'language'])
                for i in new_df:
                    writer.writerow(new_df[i])

        if preprocessing:
            with open('./clean_dataset/clean_'+name_dataset, encoding='utf-8-sig', errors='replace') as f:
                df = pd.read_csv(f, sep=',', names=['name','score','title','place','date','verified purch','text','upvotes', 'language'])
                df = df.replace({np.nan: None})
                name = df['name'].to_dict()
                score = df['score'].to_dict()
                title = df['title'].to_dict()
                place = df['place'].to_dict()
                date = df['date'].to_dict()
                ver_purchase = df['verified purch'].to_dict()
                text = df['text'].to_dict()
                upvotes = df['upvotes'].to_dict()
                language = df['language'].to_dict()
            new_text = {}
            new_title = {}
            ids = {}
            for i in tqdm(text, desc='Preprocessing dataset'):
                if i > 0:
                    ids[i] = i
                    if title[i]: new_title[i] = ' '.join(preprocessing_text(title[i], language[i]))
                    else: new_title[i] = None
                    if text[i]: new_text[i] = ' '.join(preprocessing_text(text[i], language[i]))
                    else: new_text[i] = None

            with open('./preprocessed_dataset/preprocessed_'+name_dataset, 'wt', encoding='utf-8', newline='') as of:
                writer = csv.writer(of, delimiter = ',')
                writer.writerow(['id','name','score','title','place','date','verified purch','text','upvotes', 'language'])
                for i in text:
                    if i > 0 and (new_title[i] or new_text[i]):
                        writer.writerow([ids[i], name[i], score[i], new_title[i], place[i], date[i], ver_purchase[i], new_text[i], upvotes[i], language[i]])

def cleaning_twitter_datasets():
    os.chdir("G:\Il mio Drive\Project_Data_Mining\Data\Twitter")
    datasets = [f for f in glob.glob("*.csv")]
    for name_dataset in datasets:
        start_cleaning = 0
        mode = 'wt'
        with open(name_dataset, encoding='utf-8-sig', errors='replace') as f:
            df = pd.read_csv(f, sep=',', names=['username','datatime', 'tweet_text', 'retweeted_text', 'likes', 'retweets', 'comments'])
            df = df.replace({np.nan: None})
            name = df['username'].to_dict()
            date = df['datatime'].to_dict()
            text = df['tweet_text'].to_dict()
            retweet_text = df['retweeted_text'].to_dict()
            likes = df['likes'].to_dict()
            retweets = df['retweets'].to_dict()
            comments = df['comments'].to_dict()
        print("Checking operations on dataset:'"+name_dataset+"'")
        if os.path.exists('./clean_dataset/clean_'+name_dataset):
            with open('./clean_dataset/clean_'+name_dataset, encoding='utf-8-sig', errors='replace') as f:
                df_clean = pd.read_csv(f, sep=',', names=['username','datatime', 'tweet_text', 'retweeted_text', 'likes', 'retweets', 'comments', 'language'])
                if len(name) == len(df_clean['username']): cleaning = False
                else:
                    cleaning = True
                    start_cleaning = len(df_clean['username']) - 1
                    mode = 'a'
                    print('  > Start cleaning from line '+str(start_cleaning + 1))
        else:
            cleaning = True
        if os.path.exists('./preprocessed_dataset/preprocessed_'+name_dataset) and not cleaning:
            preprocessing = False
            print('  > Dataset already cleaned and preprocessed')
        else:
            preprocessing = True
        time.sleep(1)

        if cleaning:
            with open('./clean_dataset/clean_'+name_dataset, mode, encoding='utf-8', newline='') as of:
                writer = csv.writer(of, delimiter = ',')
                if mode == 'wt': writer.writerow(['username','datatime', 'tweet_text', 'retweeted_text', 'likes', 'retweets', 'comments', 'language'])
                for i in tqdm(name, desc='Cleaning dataset'):
                    if i > start_cleaning:
                        if not name[i] or name[i].isspace(): name[i] = 'unknown'
                        new_date = re.split('T',date[i])[0]
                        new_date = re.sub("-", "", new_date)
                        language = detect_language_from_text(text[i])
                        if language == 'error':
                            print(Style.BRIGHT+Fore.RED +'\nFatal error: the computation has been stopped'+ Style.RESET_ALL)
                            return
                        writer.writerow([name[i], new_date, text[i], retweet_text[i], likes[i], retweets[i], comments[i], language])
                        time.sleep(1)

        if preprocessing:
            with open('./clean_dataset/clean_'+name_dataset, encoding='utf-8-sig', errors='replace') as f:
                df = pd.read_csv(f, sep=',', names=['username','datatime', 'tweet_text', 'retweeted_text', 'likes', 'retweets', 'comments', 'language'])
                df = df.replace({np.nan: None})
                name = df['username'].to_dict()
                date = df['datatime'].to_dict()
                text = df['tweet_text'].to_dict()
                retweet_text = df['retweeted_text'].to_dict()
                likes = df['likes'].to_dict()
                retweets = df['retweets'].to_dict()
                comments = df['comments'].to_dict()
                language = df['language'].to_dict()

            # initializing sentiment analyzer
            try:
                sia = SentimentIntensityAnalyzer()
                pass
            except LookupError:
                nltk.download('vader_lexicon')
                sia = SentimentIntensityAnalyzer()
                pass

            new_text = {}
            new_retweet_text = {}
            question = {}
            new_tweet_no_ash = {}
            new_retweet_no_ash = {}
            sentiment_analysis = {}
            for i in tqdm(text, desc='Preprocessing dataset'):
                if i > 0:
                    if retweet_text[i]:
                        new_retweet_text[i] = ' '.join(preprocessing_text(retweet_text[i], language[i], True)[0])
                        new_retweet_no_ash[i] = re.sub(r"[#|@]+[\d|\w]*", '', str(new_retweet_text[i]))
                    else:
                        new_retweet_text[i] = retweet_text[i]
                        new_retweet_no_ash[i] = retweet_text[i]
                    new_text[i], question[i] = preprocessing_text(text[i], language[i], True)
                    new_text[i] = ' '.join(new_text[i])
                    new_tweet_no_ash[i] = re.sub(r"[#|@]+[\d|\w]*", '', str(new_text[i]))
                    if question[i]:
                        sentiment_analysis[i] = 0
                    else: sentiment_analysis[i] = is_positive(sia, text[i])

            with open('./preprocessed_dataset/preprocessed_'+name_dataset, 'wt', encoding='utf-8', newline='') as of:
                writer = csv.writer(of, delimiter = ',')
                writer.writerow(['username', 'datatime', 'tweet_text', 'retweeted_text', 'likes',
                                 'retweets', 'comments', 'language', 'is a question', 'tweet wout hashtag',
                                 'retweet wout hashtag', 'sentiment analysis'])
                for i in text:
                    if i > 0: writer.writerow([name[i], date[i], new_text[i], new_retweet_text[i],
                                               likes[i], retweets[i], comments[i], language[i], question[i],
                                               new_tweet_no_ash[i], new_retweet_no_ash[i], sentiment_analysis[i]])

if __name__ == '__main__':
    cleaning_amazon_datasets()
    cleaning_twitter_datasets()


from Woodchips_StreamlabsSystem import change_points

class Redeemable():

    def __init__(self, name, description, cost, user, user_input=None, args=None):
        """

        :param name: Name of the redeemable
        :param description: What the user will see when it's redeemed.
        :param cost: The cost in woodchips.
        :param user: The user's name.
        """
        self.name = name
        self.description = description
        self.cost = cost
        self.user = user
        self.custom_function = user_input
        self.args = args

    def redeem(self):
        if change_points(self.user, self.cost):
            if self.custom_function:
                if self.args:
                    self.custom_function(self.args)
                else:
                    self.custom_function()

            return True
        else:
            return False


import json
import nltk
import logging
import pickle
from tqdm import tqdm
import argparse
import string
import os
import numpy as np
import pickle
import faiss

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=logging.INFO

                        )

logger = logging.getLogger(__name__)

class Triple_cluster():
    def __init__(self,glove_dict='glove.json'):
        self.name='triple cluster'
        self.glove = glove_dict
        self.triple_types = ['stanford_t', 'minie_t_cor', 'minie_t']
        self.text_types = ['text', 'cor_text']
        if os.path.exists(self.glove):
            self.glove = self.read(glove_dict)
            print('finish load glove data with %d words.' % len(self.glove))

    def read(self,filename):
        logger.info('load data from ' + filename)
        if '.json' in filename:
            with open(filename, 'r')as f:
                data = json.load(f)
            return data
        elif '.pkl' in filename:
            with open(filename, 'rb')as f:
                data = pickle.load(f)
            return data
        elif '.txt' in filename:
            with open(filename, 'r')as f:
                data = f.readlines()
            return data
        else:
            print('cannot read data from the file!')
            return
        return

    def triple2sen(self,tlists):
        '''
        transfer the para's triples to flatternd sentence.
        :param tlists:
        :return:
        '''
        tsens=[]
        for t in tlists:
            sen = ' '.join(t)
            tsens.append(sen)
        return tsens

    def glove_encode(self, triples, merge="mean",oov=True):
        encoded_output = []
        if merge == "mean":
            for sen in triples:
                words = sen.split(' ')
                sen_embed = []
                for word in words:
                    if word in self.glove:
                        sen_embed.append(np.asarray(self.glove[word], "float32"))
                    else:
                        if oov:
                            sen_embed.append(np.asarray(np.zeros(300), "float32"))
                        else:
                            sen_embed.append(np.asarray(np.ones(300), "float32"))
                sen_embed = np.mean(sen_embed, axis=0)
                encoded_output.append(sen_embed)
        return encoded_output

    def sibling_concatenate(self,triples, maxsize=16, normalize=True):
        '''merge the two most similar triple to one triple.'''
        triples_sens = self.triple2sen(triples)
        glove_rep_tss = self.glove_encode(triples_sens, merge="mean", oov=True)
        glove_rep_tss = np.array(glove_rep_tss)
        features = glove_rep_tss
        # normalize the feature. You can replace "features" to your own one.
        if normalize:
            row_sums = np.linalg.norm(features, axis=1)
            features = features / row_sums[:, np.newaxis]

        n, dim = features.shape[0], features.shape[1]
        # index = faiss.IndexFlatIP(dim)
        index_flat = faiss.IndexFlatL2(dim)
        res = faiss.StandardGpuResources()
        index = faiss.index_cpu_to_gpu(res, 3, index_flat)
        index.add(features)
        topk = 1
        distances, indices = index.search(features, topk + 1)
        del index
        del res
        del index_flat
        out_of_bound = maxsize - len(triples)
        if out_of_bound > 16:
            out_of_bound = out_of_bound * 2
        else:
            out_of_bound = out_of_bound + 1

        score = (2 - distances) / 2
        score = score[:, 1]
        out_score = score[np.argpartition(score, out_of_bound)]
        nowscore = (out_score[-out_of_bound])

        newtriple = []
        already = []
        for idx, ind in enumerate(indices):
            if ind[0] not in already and score[idx] >= nowscore:
                combine = (triples[idx][:-1].split(' ')) + (triples[ind[1]].split(' '))
                combine = list(dict.fromkeys(combine))  # remove redundant elements.
                newtriple.append(' '.join(combine))
                already.append(ind[1])
                already.append(ind[0])
            elif ind[0] not in already:
                newtriple.append(triples[idx])

        return newtriple

    def s_canopiese(self,triples):
        '''
        split the triples into smaller canopies by same subject.
        :param triples:
        :return:
        '''
        canopies=dict()
        for t in triples:
            subject=t[0]
            if subject not in canopies:
                canopies[subject]=[t]
            else:
                canopies[subject]=canopies[subject]+[t]
        return canopies

    def sp_canopiese(self,triples):
        '''
        split the triples into smaller canopies by same subject-predicate.
        :param triples:
        :return:
        '''
        canopies=dict()
        for t in triples:
            subject=t[0]
            predicate=t[1]
            sp=subject+'$$'+predicate
            if sp not in canopies:
                canopies[sp]=[t]
            else:
                canopies[sp]=canopies[sp]+[t]
        return canopies


#!/usr/bin/env python3

import re
import copy

rules = {}
back_rules = {}

lines = open('input.txt').read().splitlines()

for line in lines:
    parts = line.split(' ')
    lhs = parts[0] + ' ' + parts[1]
    assert lhs not in rules
    rules[lhs] = []

    line = ' '.join(parts[3:])
    for m in re.findall('(\d+) ([\w ]+) bag', line):
        rhs = m[1]
        if rhs not in back_rules:
            back_rules[rhs] = set()
        back_rules[rhs].add(lhs)
        rules[lhs].append((m[1], int(m[0])))

#part 1
start = {'shiny gold'}
while True:
    l = len(start)
    for item in list(start):
        if item in back_rules:
            start |= back_rules[item]
    if l == len(start):
        break

print(len(start) - 1)

# part 2
total = 0
bags = {'shiny gold': 1}
while bags:
    print(bags)
    k, v = list(bags.items())[0]
    total += v
    del bags[k]

    if k in rules:
        print(f'{v}x {k} => {rules[k]} (total: {total})')
        for k2, v2 in rules[k]:
            bags.setdefault(k2, 0)
            bags[k2] += v * v2
print(total - 1)


# Marcelo Campos de Medeiros
# ADS UNIFIP 2020.1
# Patos-PB 20/03/2020

'''
Faça um programa que calcule as raízes de uma equação do segundo grau,
na forma ax2 + bx + c. O programa deverá pedir os valores de a, b e c
e fazer as consistências, informando ao usuário nas seguintes situações:
    Se o usuário informar o valor de A igual a zero, a equação não é do segundo grau
    e o programa não deve fazer pedir os demais valores, sendo encerrado;
    Se o delta calculado for negativo, a equação não possui raizes reais.
        Informe ao usuário e encerre o programa;
    Se o delta calculado for igual a zero a equação possui apenas uma raiz real;
        informe-a ao usuário;
    Se o delta for positivo, a equação possui duas raiz reais;
        informe-as ao usuário;
'''

print('='*50)
print('{:=^40}'.format(" PROGRAMA QUE CALCULE AS RAÍZES DA EQUAÇÃO 2° GRAU "))
print('='*50, '\n')

print('EQUAÇÃO: ax² + bx + c')
# variável ax² + bx + c, informar as variéis: a, b, c
a = int(input('Dentro da equação acima informe a variável "a": '))
# condição caso "a" seja == 0
if a == 0:
    print('Programa encerrado!!!\n'
          'Valor não satisfaz a condição para ser uma equação do 2° grau')
# "a" != 0
else:
    b = int(input('Dentro da equação acima informe a variável "b": '))
    c = int(input('Dentro da equação acima informe a variável "c": '))
    delta = b ** 2 - 4 * a * c
    # delta calculado for negativo
    if delta < 0:
        print('\n', 'Programa encerrado!!!\n'
              f'Valor de delta é negativo {delta}\n'
              f' portanto a equação não possui raizes reais')
    elif delta == 0:
        x = (-b) / (2 * a)
        print('\n', 'Valor de delta é igual a "0"\n'
              f'Portanto a equação possui apenas uma raiz real que é {x}')
    else:
        x_1 = ((-b) + delta) / (2 * a)
        x_2 = ((-b) - delta) / (2 * a)
        print('\n', 'Valor de delta é positivo\n'
              f'Portanto a equação possui duas raiz reais que são {x_1:.2f} e {x_2:.2f}')


import shutil
import socket
import os
'''
pwd - показывает название рабочей директории
ls - показывает содержимое текущей директории
cat <filename> - отправляет содержимое файла
'''

dirname = os.path.join(os.getcwd(), 'docs')

def process(req):
    #Где мы находимся(путь)
    if req == 'pwd':
        return dirname

    #Просмтариваем файлы и папки
    elif req == 'ls':
        return '; '.join(os.listdir(dirname))

    #Просматриваем файл
    elif req[:3] == 'cat':
        path = os.path.join(os.getcwd(), 'docs', req[4::])
        if os.path.exists(path):
                with open(path, 'r+') as file:
                    line = ''
                    for l in file:
                        line+=l
                return line
        else:
            return 'Файла не существует'

    #Создаем папку
    elif req[:5] == 'mkdir':
        path = os.path.join(os.getcwd(), 'docs', req[6:])
        if not os.path.exists(path):
            os.makedirs(path, exist_ok=True)
            return f'Папка {req[6:]} создана!'
        else:
            return f'Неверный ввод {req[6:]}'

    #Рекурсивное удаление пустой папки
    elif req[:5] == 'rmdir':
        if req[5] == " ":
            path = os.path.join(os.getcwd(), 'docs', req[6:])
            if os.path.exists(path):
                os.rmdir(path)
                return f'Папка {req[6:]} удалена'
            else:
                return f'Папки {req[6:]} не существует'
        else:
            return f'Команда введена неверно!'

    #Создание файла
    elif req[:6] == 'create':
        path = os.path.join(os.getcwd(), 'docs', req[7:])
        open(path,'tw', encoding='utf-8')
        return f'Файл {req[7:]} был создан!'

    #Удаление файла
    elif req[:6] == 'remove':
        path = os.path.join(os.getcwd(), 'docs', req[7:])
        try:
            os.remove(path)
            return f'Файл {req[7:]} был удален!'

        except:
            return f'Файл не был найден'

    #Переименуем файл
    elif req[:6] == 'rename':
        if req[6] == " ":
            try:
                req_s = req.split(' ')
                path1 = os.path.join(os.getcwd(), 'docs', req_s[1])
                path2 = os.path.join(os.getcwd(), 'docs', req_s[2])
                os.rename(path1, path2)
                return f'Файл {req_s[1]} был переименован в {req_s[2]}'

            except:
                return f'Не в этот раз'

    #Копирование файла
    elif req[:4]  == 'copy':
        if req[4] == " ":
            try:
                req_s = req.split(' ')
                path1 = os.path.join(os.getcwd(), 'docs', req_s[1])
                path2 = os.path.join(os.getcwd(), 'docs', req_s[2])
                shutil.copyfile(path1, path2, follow_symlinks=True)
                return f'Файл {req_s[1]} скопирован!'

            except:
                f'Копирование прошло неудачно, попробуйте ввести корректное имя файла!'

    return 'bad request'

PORT = 6699

sock = socket.socket()
sock.bind(('', PORT))
sock.listen()
print("Прослушиваем порт", PORT)

while True:
    conn, addr = sock.accept()

    request = conn.recv(1024).decode()
    print(request)

    response = process(request)
    conn.send(response.encode())

conn.close()


r"""
Run tracker evaluation on existing tracker results.

Re-evaluate existing tracker results, e.g. with a different matching algorithm.
Designed for use with run_tracker_5_games.py or run_tracker_5_durations.py.

Requires the files tracker-settings.json and resutls/tracks.csv to be present
in the directory structure. Can accept a directory path to either parentdir or
gparentdir:
    results/pftracker/
        gparentdir/
            parentdir/
                tracker-settings.json
                results/tracks.csv

Usage:
    python3 run_tracker_reevaluation.py \
        --dir path/to/tracking/results/ \
        --algorithm "particle filter" \
        --match "dtw-sliding-window"
"""

import argparse
import datetime
import json
import os

import numpy as np
import matplotlib.pyplot as plt

from src.tracking import tracker_util
from src.tracking.fp_tracker import ForwardsPassTracker
from src.tracking.pf_tracker import ParticleFilterTracker
from src.scenario.mot_scenario import MOT_Scenario

def check_directory(out_dir):
    """
    Check directory contains the appropriate assets for tracker re-evaluation.

    Directory must contain the file results/object_tracker_results.csv for it
    to be re-evaluated.

    Args:
        out_dir         Directory containing tracker output
    Returns:
        assets_present  Boolean value indicating whether the required assets
                        are present in the directory
    """
    assets_present = False
    if os.path.exists(os.path.join(
            out_dir, "results/object_tracker_results.json")):
        assets_present = True
    return assets_present

def reevaluate_tracks(
        base_out_dir, match_type, position_set_id, new_out_dir, reuse,
        algorithm):
    """
    Re-evaluate tracks using specified match, position set(s).

    Args:
        base_out_dir        Path to directory containing tracker_settings.json
                            and  results/tracks.csv
        match_type          Matching algorithm to use for evaluation
        position_set_id     Position set ID/s to use for evaluation
        new_out_dir         Path to directory to save results
        reuse               Flag to re-using object detector
        algorithm           Tracker algorithm to use
    """
    timing_record = {"start_time": "{0}".format(datetime.datetime.now())}
    # Output directory:
    if not os.path.exists(new_out_dir):
        os.makedirs(new_out_dir)
    # Load tracks:
    tracker_type, file_generation_time, tracker_settings, tracks, \
        track_bboxes = tracker_util.load_object_tracker_results_from_json(
            os.path.join(
                base_out_dir, "results/object_tracker_results.json"))
    # Correct camera calibration IDs stored in video info:
    for item in tracker_settings["video_info"]:
        item[1] = tuple(item[1])
    # Set up scenario:
    scenario = MOT_Scenario(
        scenario=tracker_settings["scenario_name"],
        data_root=tracker_settings["scenario_data_root"])
    # Set up tracker:
    if algorithm == "particle filter":
        tracker = ParticleFilterTracker(
            allow_run_detector=True, reuse=reuse,
            tracker_frequency=tracker_settings["tracker_frequency"],
            n_particles=tracker_settings["n_particles"],
            init_sigma_p=tracker_settings["init_sigma_p"],
            init_sigma_v=tracker_settings["init_sigma_v"],
            sigma_p=tracker_settings["sigma_p"],
            sigma_v=tracker_settings["sigma_v"],
            weight_sigma_p=tracker_settings["weight_sigma_p"],
            ess_threshold_proportion=tracker_settings[
                "ess_threshold_proportion"],
            max_detection_distance=tracker_settings["max_detection_distance"],
            max_latent_time=tracker_settings["max_latent_time"],
            min_track_distance=tracker_settings["min_track_distance"],
            trailing_duration=tracker_settings["trailing_duration"])
    elif algorithm == "forwards pass":
        tracker = ForwardsPassTracker(
            tracker_frequency=4, reuse=reuse,
            tracking_window=tracker_settings["tracking_window"],
            grid_interval=tracker_settings["grid_interval"],
            detection_kernel_sigma=tracker_settings["detection_kernel_sigma"],
            detection_kernel_radius=tracker_settings[
                "detection_kernel_radius"],
            max_speed=tracker_settings["max_speed"],
            max_sq_dist_to_minima=tracker_settings["max_sq_dist_to_minima"],
            min_minima_prop=tracker_settings["min_minima_prop"],
            minima_filter_multiplier=tracker_settings[
                "minima_filter_multiplier"]
        )
    else:
        raise ValueError("invalid algorithm type: {}".format(algorithm))
    if reuse is None:
        reuse = True
    # Set tracking times, field, match type:
    tracker.set_tracking_times(
        datetime.datetime.strptime(
            tracker_settings["utc_start_time"], "%Y-%m-%d %H:%M:%S"),
        tracker_settings["tracking_duration"])
    tracker.set_field(scenario)
    tracker.match_type = match_type
    timing_record["set_up"] = "{0}".format(datetime.datetime.now())
    timing_record["tracking"] = "{0}".format(datetime.datetime.now())
    # Evaluate tracks:
    tracker.write_settings_to_json(
        scenario_name=scenario.name, scenario_data_root=scenario.data_root,
        video_info=tracker_settings["video_info"], out_dir=new_out_dir)
    scenario.add_position_set(
        "direct", world_data=tracks, position_set_identifier='tracker',
        utc_reference_time=tracker.utc_start_time,
        field_json_path=scenario.field_json_path)
    tracker.evaluate_and_save(
        os.path.join(new_out_dir, "results/"), tracks, scenario,
        tracker_settings["video_info"], position_set_id,
        track_bboxes=track_bboxes, match_type=match_type, plot_per_track=False)
    timing_record["evaluate_and_save"] = "{0}".format(datetime.datetime.now())
    timing_record["postprocessing"] = "{0}".format(datetime.datetime.now())
    timing_record["pp_evaluate_and_save"] = "{0}".format(
        datetime.datetime.now())
    with open(os.path.join(new_out_dir, "timings.json"), "w") as jf:
        json.dump(timing_record, jf, indent=4, sort_keys=True)
    return reuse

if __name__ == "__main__":
    # Unpack input:
    ap = argparse.ArgumentParser()
    ap.add_argument(
        "-d", "--dir", required=True, type=str,
        help="local directory path containing tracking results")
    ap.add_argument(
        "-a", "--algorithm", required=False, type=str,
        help="tracking algorithm", default="particle filter",
        choices=["particle filter", "forwards pass"])
    ap.add_argument(
        "-m", "--match", required=False, type=str,
        help="matching algorithm", default="shape-sliding-window",
        choices=["position", "shape-sliding-window", "dtw-sliding-window",
                 "displacement"])
    args = vars(ap.parse_args())
    base_out_dir = args["dir"]
    algorithm = args["algorithm"]
    match_type = args["match"]

    print(
        "Re-evaluating tracks in {0} using the {1} tracker with {2} "
        "matching".format(base_out_dir, algorithm, match_type))

    # Evaluation information:
    position_set_id = [
        "h1-radar.json-primary", "h1-radar.json-linked",
        "h2-radar.json-primary", "h2-radar.json-linked"]

    reuse = None
    if check_directory(base_out_dir):
        print("Re-evaluating data in {0}...".format(base_out_dir))
        base_parent_dir, base_dirname = os.path.split(base_out_dir)
        if os.path.samefile(base_parent_dir, base_out_dir):
            base_parent_dir, base_dirname = os.path.split(base_parent_dir)
        new_out_dir = os.path.join(
            base_parent_dir, "{0}-{1}".format(base_dirname, match_type))
        reuse = reevaluate_tracks(
            base_out_dir, match_type, position_set_id, new_out_dir, reuse)
    else:
        items = sorted(os.listdir(base_out_dir))
        base_parent_dir, base_dirname = os.path.split(base_out_dir)
        if os.path.samefile(base_parent_dir, base_out_dir):
            base_parent_dir, base_dirname = os.path.split(base_parent_dir)
        new_out_dir = os.path.join(
            base_parent_dir, "{0}-{1}".format(base_dirname, match_type))
        for cur_item in items:
            cur_dir = os.path.join(base_out_dir, cur_item)
            cur_out_dir = os.path.join(new_out_dir, cur_item)
            if os.path.isdir(cur_dir) and check_directory(cur_dir):
                print("Re-evaluating data in {0}...".format(cur_dir))
                reuse = reevaluate_tracks(
                    cur_dir, match_type, position_set_id, cur_out_dir, reuse,
                    algorithm)


import time
import zipfile
import io

def put_watermark(epub, watermark):
    """
    Append watermark and timestamp as comment to given epub file

    :param epub: in-memory file object representation of ebook
    :param watermark: str code
    :return: file object with watermark
    """
    MANIFEST_FILENAME = 'META-INF/container.xml'

    with zipfile.ZipFile(epub, 'r') as zip_in:
        new_epub = io.BytesIO()
        with zipfile.ZipFile(new_epub, 'w') as zip_out:
            zip_out.comment = zip_in.comment
            # ZipFile can't update existing files,
            # so we must create new one and copy all files except file needed to update
            for item in zip_in.infolist():
                if item.filename != MANIFEST_FILENAME:
                    zip_out.writestr(item, zip_in.read(item.filename))

            manifest = zip_in.read(MANIFEST_FILENAME)
            manifest = '%s\n<!-- %s %s -->' % (manifest, watermark, time.time()) # actually append watermark

            zip_out.writestr(MANIFEST_FILENAME, bytes(manifest))
    return new_epub


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 16 10:38:19 2017

@author: gopi
"""

s = str(input('Enter the String for which vowels to be counted: '))
numOfVowels = 0
for char in s:
    if char =='a' or char == 'e' or char == 'i' or char == 'o' or char == 'u':
        numOfVowels+=1
print('Number of vowels:',numOfVowels)


from __future__ import absolute_import, division, print_function

__all__ = [
    "__title__", "__summary__", "__uri__", "__version__", "__author__",
    "__email__", "__license__", "__copyright__",
]

__title__ = "historical_reports_s3"
__summary__ = "Historical function for generating full dictionary of all S3 buckets and corresponding metadata."
__uri__ = "https://github.com/Netflix-Skunkworks/historical-reports"

__version__ = "0.1.1"

__author__ = "The Historical developers"
__email__ = "security@netflix.com"

__license__ = "Apache License, Version 2.0"
__copyright__ = "Copyright 2017 {0}".format(__author__)


from dataclasses import dataclass
import datetime
from typing import Dict, List

@dataclass()
class Match:
    _id: int
    line_group_id: str
    hanchan_ids: List[int]
    users: list
    status: int
    tip_scores: Dict[str, int]
    created_at: datetime.date

    def __init__(
        self,
        line_group_id: str,
        hanchan_ids: List[int],
        status: int,
        users: list = [],
        created_at: datetime.date = None,
        tip_scores: Dict[str, int] = {},
        _id: int = None,
    ):
        self._id = _id
        self.line_group_id = line_group_id
        self.hanchan_ids = hanchan_ids
        self.users = users
        self.status = status
        self.tip_scores = tip_scores
        self.created_at = created_at

# status は 0: disabled, 1: active, 2: archived

# created_at は対戦開始日


import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

'''
This is my implementation of the densely connected fully convolutional network aka densenet.
The paper describing the net can be found at:
http://arxiv.org/abs/1608.06993
The caffe model that I took from was implemented at:
https://github.com/liuzhuang13/DenseNetCaffe/blob/master/make_densenet.py

Main model hyperparameters:
    depth - The total number of conv/pool layers in the model, must equal
        2 (i/o) + 1 * [num_blocks-1] (transition) + num_blocks * block_layers (block layers)
        In our default case, 7 = 2 + 1 * [2-1] + 2 * 2
    first_output - number of layers in our initial convolution
    growth_rate - the number of additional new filters to concatenate with our previous layer

'''
#Basic functions for initializing layers
def weights_init(shape):
    return tf.Variable(tf.truncated_normal(shape, seed=1))

def bias_init(shape):
    return tf.Variable(tf.constant(0.1))

def conv2d(x, w):
    return tf.nn.conv2d(x, w, strides=[1,1,1,1], padding='SAME')

def maxpool(x):
    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding="VALID")

def dropout(x, keep):
    return tf.nn.dropout(x, keep, seed=1)

##The slightly more complicated blocks of layers

#Basic block of the model
#Takes in a convolution layer, adds batch norm, relu, and a new convolution with dropout
def bn_relu_conv(inputs, w_shape, b_shape, keep, is_train):
    w = weights_init(w_shape)
    batch = tf.contrib.layers.batch_norm(inputs, is_training=is_train)
    relu = tf.nn.relu(batch)
    conv = conv2d(relu, w)
    drop = dropout(conv, keep)
    return drop

#Adds a batch norm/relu/conv layer and concatenates it with the input layer
def add_layer(bottom, num_filter, keep, is_train):
    num_old = int(bottom.get_shape()[3])
    brc = bn_relu_conv(bottom, [3,3,num_old,num_filter], [num_filter], keep, is_train)
    concat = tf.concat(3, [bottom, brc])
    return concat

#The transition layer between blocks
#Performs a bn/relu/conv and then a 2x2 maxpool to reduce the image size
def transition(bottom, num_filter, keep, is_train):
    num_old = int(bottom.get_shape()[3])
    brc = bn_relu_conv(bottom, [3,3,num_old,num_filter], [num_filter], keep, is_train)
    pool = maxpool(brc)
    return pool

#This portion puts the model together through the use of looping
#I've sized it down from the original paper to fit on the smaller GPUs available
def pred(x, keep, is_train, depth=40, first_output=16, growth_rate=12):

    n_classes = 10
    n_channels = first_output
    #First conv layer that starts the model
    with tf.variable_scope("input"):
        w = weights_init([3,3,3,n_channels])
        layer = conv2d(x, w)

    N = (depth - 4)/3

    for i in range(1,N+1):
        name = "block1-{}".format(i)
        with tf.variable_scope(name):
            layer = add_layer(layer, n_channels, keep, is_train)
        n_channels += growth_rate
    with tf.variable_scope("trans1"):
        layer = transition(layer, n_channels, keep, is_train)

    for i in range(1,N+1):
        name = "block2-{}".format(i)
        with tf.variable_scope(name):
            layer = add_layer(layer, n_channels, keep, is_train)
        n_channels += growth_rate
    with tf.variable_scope("trans2"):
        layer = transition(layer, n_channels, keep, is_train)

    for i in range(1,N+1):
        name = "block3-{}".format(i)
        with tf.variable_scope(name):
            layer = add_layer(layer, n_channels, keep, is_train)
        n_channels += growth_rate

    with tf.variable_scope("output"):
        batch = tf.contrib.layers.batch_norm(layer, is_training=is_train)
        relu = tf.nn.relu(batch)
        rs = relu.get_shape()
        n_flat = int(rs[1])*int(rs[2])*int(rs[3])
        flat = tf.reshape(relu, [-1, n_flat])
        w = weights_init([n_flat, n_classes])
        b = bias_init([n_classes])
        dense = tf.matmul(flat, w)
        output = tf.nn.softmax(dense + b)

    return output

def acc(y, y_pred):
    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_pred,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    return accuracy


def read_dict(filename, noun, num = 'None'):
    with open('freq_dict.txt', encoding='utf-8') as f:
	    t = f.read()
	    lines = t.splitlines()
    lines_noun = []
    for line in lines:
        if 'сущ' in line:
            lines_noun.append(line)
    lil = ' '.join(lines_noun)
    aota = []
    word, part_sp, _ = lil.split('|')
    out = (word, part_sp)
    return auta.append(out)
print(read_dict('freq_dict.txt', 'сущ'))


import os
import shutil
import subprocess
import sys
import tempfile
import _thread as thread
try:
    from netranger.Vim import VimErrorMsg as CmdFailLog
except Exception:
    CmdFailLog = print

def log(*msg):
    with open(os.path.join(tempfile.gettempdir(), "netlog"), 'a') as f:
        f.write(' '.join([str(m) for m in msg])+"\n")

def spawnDaemon(func):
    # do the UNIX double-fork magic, see Stevens' "Advanced
    # Programming in the UNIX Environment" for details (ISBN 0201563177)
    try:
        pid = os.fork()
        if pid > 0:
            # parent process, return and keep running
            return
    except OSError as e:
        print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror)
        sys.exit(1)

    os.setsid()

    # do second fork
    try:
        pid = os.fork()
        if pid > 0:
            # exit from second parent
            sys.exit(0)
    except OSError as e:
        print >>sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror)
        sys.exit(1)

    # do stuff
    func()

    # all done
    os._exit(os.EX_OK)

class Shell():
    userhome = os.path.expanduser('~')

    @classmethod
    def ls(cls, dirname):
        return os.listdir(dirname)

    @classmethod
    def abbrevuser(cls, path):
        return path.replace(Shell.userhome, '~')

    @classmethod
    def run(cls, cmd):
        try:
            return subprocess.check_output(cmd,
                                           shell=True,
                                           stderr=subprocess.STDOUT).decode('utf-8')
        except subprocess.CalledProcessError as e:
            CmdFailLog(e)

    @classmethod
    def run_async(cls, cmd):
        thread.start_new_thread(lambda: Shell.run(cmd), ())

    @classmethod
    def touch(cls, name):
        Shell.run('touch "{}"'.format(name))

    @classmethod
    def rm(cls, name):
        Shell.run('rm -r ' + name)

    @classmethod
    def spawn(cls, cmd):
        spawnDaemon(lambda: subprocess.check_output(cmd.split(' ')).decode('utf-8'))

    @classmethod
    def shellrc(cls):
        return os.path.expanduser('~/.{}rc'.format(os.path.basename(os.environ['SHELL'])))

    @classmethod
    def cp(cls, src, dst):
        shutil.copy2(src, dst)

    @classmethod
    def mkdir(cls, name):
        if not os.path.isdir(name):
            os.makedirs(name)

    @classmethod
    def chmod(cls, fname, mode):
        os.chmod(fname, mode)

    @classmethod
    def isinPATH(cls, exe):
        return any(os.access(os.path.join(path, exe), os.X_OK) for path in os.environ["PATH"].split(os.pathsep))

    @classmethod
    def urldownload(cls, url, dst):
        import sys
        if sys.version_info[0] < 3:
            import urllib2 as urllib
        else:
            import urllib.request as urllib

        hstream = urllib.urlopen(url)
        with open(dst, 'wb') as f:
            f.write(hstream.read())

def c256(msg, c, background):
    if background:
        return '[38;5;{};7m{}'.format(c, msg)
    else:
        return '[38;5;{}m{}'.format(c, msg)


"""
Given a pattern and a string str, find if str follows the same pattern.

Here follow means a full match, such that there is a bijection
between a letter in pattern and a non-empty substring in str.

Examples:
pattern = "abab", str = "redblueredblue" should return true.
pattern = "aaaa", str = "asdasdasdasd" should return true.
pattern = "aabb", str = "xyzabcxzyabc" should return false.

Notes:
You may assume both pattern and str contains only lowercase letters.

"""

class Solution1(object):
    def wordPatternMatch(self, pattern, s):
        """
        :type pattern: str
        :type str: str
        :rtype: bool
        """
        def isMatch(pattern, i, word, j, dic, strs):
            if i == len(pattern) and j == len(word): return True
            if i == len(pattern) or j == len(word): return False

            c = pattern[i]
            if c in dic:
                s = dic[c]
                if s != word[j:j + len(s)]: return False
                return isMatch(pattern, i + 1, word, j + len(s), dic, strs)

            for k in range(j, len(word)):
                s = word[j:k + 1]
                if s in strs: continue
                dic[c] = s
                strs.add(s)
                if isMatch(pattern, i + 1, word, j + len(s), dic, strs): return True
                strs.remove(s)
                dic.pop(c)
            return False

        return isMatch(pattern, 0, s, 0, {}, set())

class Solution(object):
    def wordPatternMatch(self, pattern, s):
        """
        :type pattern: str
        :type str: str
        :rtype: bool
        """
        def isMatch(pattern, i, word, j, dic, strs):
            if i == len(pattern) and j == len(word): return True
            if i == len(pattern) or j == len(word): return False

            c = pattern[i]
            if dic[ord(c)]:
                s = dic[ord(c)]
                if s != word[j:j + len(s)]: return False
                return isMatch(pattern, i + 1, word, j + len(s), dic, strs)

            for k in range(j, len(word)):
                s = word[j:k + 1]
                if s in strs: continue
                dic[ord(c)] = s
                strs.add(s)
                if isMatch(pattern, i + 1, word, j + len(s), dic, strs): return True
                strs.remove(s)
                dic[ord(c)] = ""
            return False

        return isMatch(pattern, 0, s, 0, [""] * 256, set())


from turtle import Screen, Turtle
from paddle import Paddle
from ball import Ball
from scoreboard import Scoreboard
import time

screen = Screen()
screen.bgcolor("black")
screen.setup(800, 600)
screen.title("Pong")
screen.tracer(0)

paddle_1 = Paddle(-350, 0)
paddle_2 = Paddle(350, 0)
ball = Ball()
scoreboard = Scoreboard()

screen.listen()
screen.onkey(paddle_1.go_up, "w")
screen.onkey(paddle_1.go_down, "s")
screen.onkey(paddle_2.go_up, "Up")
screen.onkey(paddle_2.go_down, "Down")

game_is_on = True

while game_is_on:
    time.sleep(ball.move_speed)
    ball.move()
    screen.update()

    if ball.ycor() > 280 or ball.ycor() < -280:
        ball.bounce_y()

    if ball.distance(paddle_2) < 50 and ball.xcor() > 320 or ball.distance(paddle_1) < 50 and ball.xcor() > -320:
        ball.bounce_x()
        print("WE HIT")

    if ball.xcor() < -380:
        ball.reset_position()
        scoreboard.increase_p2_score()

    if ball.xcor() > 380:
        ball.reset_position()
        scoreboard.increase_p1_score()

screen.exitonclick()


from collections import deque

# Input: s = "A man, a plan, a canal: Panama"
# Output: true
# Explanation: "amanaplanacanalpanama" is a palindrome.
# Example 2:
#
# Input: s = "race a car"
# Output: false
# Explanation: "raceacar" is not a palindrome.

# palindrome
# class Solution:
#     def isPalindrome(self, s: str) -> bool:
#         check = deque()
#         for char in s:
#             if char.isalnum():
#                 check.append(char.lower())
#
#         while len(check) > 1:
#             if check.popleft() != check.pop():
#                 return False
#         return True
#
# print(Solution().isPalindrome("race a car"))

#reverse String

# Input: s = ["h","e","l","l","o"]
# Output: ["o","l","l","e","h"]

s = ["H","a","n","n","a","h"]
# Output: ["h","a","n","n","a","H"]

from typing import List
#
#
# class Solution:
#     def reverseString(self, s: List[str]) -> None:
#         s.reverse()
#둘중 하나 선택!
#         s[:] = s[::-1]
#         return s
#         # return s[::-1]
#
# print(Solution().reverseString(s))

#Reorder Data in Log Files

logs = ["dig1 8 1 5 1","let1 art can","dig2 3 6","let2 own kit dig","let3 art zero"]
# Output: ["let1 art can","let3 art zero","let2 own kit dig","dig1 8 1 5 1","dig2 3 6"]
# Explanation:
# The letter-log contents are all different, so their ordering is "art can", "art zero", "own kit dig".
# The digit-logs have a relative order of "dig1 8 1 5 1", "dig2 3 6".

# Input: logs = ["a1 9 2 3 1","g1 act car","zo4 4 7","ab1 off key dog","a8 act zoo"]
# Output: ["g1 act car","a8 act zoo","ab1 off key dog","a1 9 2 3 1","zo4 4 7"]

class Solution:
    def reorderLogFiles(self, logs: List[str]) -> List[str]:
        letter, digits = [], []
        for make_list in logs:
            # print(make_list[1:])
            if make_list.split()[1].isdigit():
                digits.append(make_list)
            else:
                letter.append(make_list)

        letter.sort(key=lambda x: [x.split()[1:], x.split()[0]])
        return letter + digits

print(Solution().reorderLogFiles(logs))


# -*- coding: utf-8 -*-
"""
Created on Thu Oct  8 18:45:42 2020

@author: Benjamin
"""

import os
import pickle

from fatiando.vis import mpl #, myv
from fatiando import gridder, mesher, utils
from fatiando.gravmag import prism, imaging, transform
from fatiando.vis.mpl import square

# my own functions
import lib.dEXP as dEXP
from lib.dEXP import _fit
import lib.plot_dEXP as pEXP
import lib.utils_dEXP as uEXP

import notebooks_GRL.load_MALM_model as MALMmod
import notebooks_GRL.load_MALM_real as MALMreal
from mpl_axes_aligner import align

import lib.set_parameters as para

import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['font.size'] = 15

# icsd functions
#from importers.read import load_obs, load_geom

#%% ------------------------------- MALM DATA synthetic

interp_size = 300
smooth = 'CubicSmoothingSpline' #'Hanning+Lowpass'
# smooth = 'CubicSmoothingSpline + interp1d' #'Hanning+Lowpass'
interp = False
file = 'NoAno'

path =  './data/phNO/'
dataset = MALMmod.load_MALM_Landfill_model(path=path,
                                    filename=file,
                                    shape = (300,300),
                                    field=True,
                                    interp = interp,
                                    radius=60) # length of p1p2 profile

coord_xyz, coord_xyz_int = dataset[0:2]
coord_xyz_int
Uload = dataset[2]
coords_liner = dataset[3]
shape, max_elevation = dataset[4:6]

dict_data = dataset[7]
dict_data['AnoBool']

xA = (dict_data['HZ'][0][0]+dict_data['HZ'][0][1])/2
x1 = dict_data['HZ'][0][2]
y1 = dict_data['HZ'][0][0]
y2 = dict_data['HZ'][0][1]

z1 = dict_data['HZ'][1]
z2 = z1 - dict_data['HZ'][2]

p = dataset[6]         # line points
# set imaging pseudo-inversion parameters
parameters = para.set_par(shape=shape,max_elevation=max_elevation)

scaled = parameters[0]
SI = parameters[1]
zp, qorder, nlay = parameters[2:5]
minAlt_ridge, maxAlt_ridge = parameters[5:7]

max_elevation = 30
# nlay = 50

# xp, yp, zp = coord_xyz_int
xp, yp, zp = coord_xyz
# len(xp)
Uini = Uload[0] # U_raw, Ucor, U_int, Ucor_int
p1 , p2 = p

rot = 60
origin=(max(xp), min(yp))
point_torotate = np.array([xp, yp])
xp_r, yp_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)
Xs = xp_r-min(xp_r)
Ys = yp_r-min(yp_r)

prl = 60
# shape = shape  (max(xp)-min(xp))/
shape = (150,150)
xint_scipy, yint_scipy = gridder.regular((min(Xs)-prl, max(Xs)+prl,
                          min(Ys)-prl, max(Ys)+prl),shape=shape)

#%% ------------------------------- MALM DATA real
MainPath= './data/'
# os.chdir(MainPath)
## --------- read MALM measured data file (with electrode positions) --------- ##
# RealData = np.loadtxt("./1_Data_2_plot/to_plot.dat",skiprows=0,delimiter='\t')
out = MALMreal.load_MALM_Porto_real(MainPath,
                          MainPath + 'XYObs_real_f_m3.txt',
                          shape=(100,100),
                          radius=200,
                          rcor=10,
                          rot=0,
                          showfig=False)

# coords_liner = out[3]
p = out[6]         # line points
p1f , p2f = p
coord_xyz, coord_xyz_int = out[0:2]
xf, yf, zf = coord_xyz
# len(xp)

out_r = MALMreal.load_MALM_Porto_real(MainPath,
                          MainPath + 'XYObs_real_f_m3.txt',
                          shape=(100,100),
                          radius=200,
                          rcor=10,
                          rot=60,
                          showfig=False)

# coords_liner = out[3]
p = out_r[6]         # line points
p1f_r , p2f_r = p
coord_xyz_r, coord_xyz_int_r = out_r[0:2]
Uload_r = out_r[2]
uf = Uload_r[1] # U_raw, Ucor, U_int, Ucor_int
# xf_rtmp, yf_rtmp, zf_rtmp = coord_xyz

# plt.figure()
# plt.scatter(xf_rtmp, yf_rtmp, c=uf, cmap='viridis',vmax=0.05)
# plt.colorbar()
# plt.axis('square')
# plt.show()

#%% Mirror field against p1p2

# find point position with respect to line equation defined by p1 and p2
U_af, p_af, bool_abovef, U_bf, p_bf = MALMmod.isabove(xf, yf, uf,
                                  np.array(p1),np.array(p2))
# Compute p1 and p2 line equation ax + by + c = 0
af, bf, cf = MALMmod.slope(p1,p2)

# Mirror points with respect to p1p2 line
Umirrorf, xy_mirrorf = MALMmod.mirrorU_alongLine(U_af,p_af,bool_abovef,af,bf,cf)

U_a_intf = gridder.interp_at(xy_mirrorf[:,0], xy_mirrorf[:,1], Umirrorf, xf, yf, algorithm='cubic',
                        extrapolate=True)
# U_mirror_int = np.copy(U_a_int)
U_mirror_intf = np.copy(uf)
U_mirror_intf[np.where(bool_abovef == True)[0]]= U_a_intf[np.where(bool_abovef == True)[0]]

xf_mirror = np.hstack([xy_mirrorf[:,0],p_af[:,0]])
yf_mirror = np.hstack([xy_mirrorf[:,1],p_af[:,1]])
uf_mirror = np.hstack([Umirrorf,U_af])

#%% choose raw or mirrored field

Uf = np.copy(uf)

Uf  = np.copy(uf_mirror)
xf = xf_mirror
yf = yf_mirror

# %% rotate and rescale all

rot = 60
origin=(max(xp), min(yp))
# origin=(max(Xf), min(Yf))
point_torotate = np.array([xp, yp])
xp_r, yp_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)

point_torotate = np.array([xf, yf])
xf_r, yf_r = MALMreal.rotate_60(origin, point_torotate, angle_deg=rot, showfig=False)
Xfs = xf_r-min(xp_r)
Yfs = yf_r-min(yp_r)

plt.figure()
plt.scatter(xf_r, yf_r, c=Uf, cmap='viridis',vmax=0.0175)
plt.colorbar()
plt.axis('square')
plt.show()

# print(Uf[0:2])

point_torotate = np.array([[dict_data['HZ'][0][0],dict_data['HZ'][0][1]],
                          [dict_data['HZ'][0][2],dict_data['HZ'][0][2]]])
xA_r, yA_r =  MALMreal.rotate_60(origin,point_torotate,angle_deg=rot, showfig=False) #58.511
xA_r = xA_r-min(xp_r)
yA_r = yA_r-min(yp_r)

point_torotate = np.array(coords_liner).T
coords_linerx, coords_linery =  MALMreal.rotate_60(origin,point_torotate,angle_deg=rot, showfig=False)
coords_linerx = coords_linerx-min(xp_r)
coords_linery = coords_linery-min(yp_r)
coords_liner_s = np.array([coords_linerx, coords_linery]).T

plt.figure()
plt.scatter(Xfs, Yfs, c=Uf, cmap='viridis',vmax=0.05)
plt.colorbar()
plt.axis('square')
plt.show()

# %%
# prl = 10
# # shape = shape  (max(xp)-min(xp))/
# shape = (30,30)
# xint_scipy, yint_scipy = gridder.regular((min(Xfs)-prl, max(Xfs)+prl,
#                           min(Yfs)-prl, max(Yfs)+prl),shape=shape)
#%% Solution 1
# extrapolate False and fill with 0 before derivative - mask them later on
U_int_scipyf = gridder.interp_at(Xfs,Yfs,Uf, xint_scipy, yint_scipy, algorithm='cubic', extrapolate=False)
InterpData = np.array([xint_scipy, yint_scipy, U_int_scipyf]).T
where_are_NaNs = np.isnan(InterpData)
InterpData[where_are_NaNs] = 0.0074
xint_scipy, yint_scipy, U_int_scipyf = InterpData.T

#%% Solution 2

# U_int_scipyf = gridder.interp_at(Xfs,Yfs, Uf, xint_scipy, yint_scipy, algorithm='cubic', extrapolate=True)
# len(U_int_scipyf)
# len(Uf)

#%% Smoothing 2d

XFs, YFs = xint_scipy,yint_scipy
UF = U_int_scipyf
UF = dEXP.smooth2d(XFs, YFs, UF, sigma=2)
# plt.savefig('smooth2d' + str(file) + '.png', dpi=450)
UFs = UF

#%%

plt.figure()
plt.scatter(XFs, YFs, c=UF, cmap='viridis')
plt.colorbar()
plt.axis('square')
plt.show()

offset = 255 #260
p1_s = np.array([p1f_r[0] -min(xp_r)+offset,p2f_r[0] -min(xp_r)+offset])
p2_s = np.array([p1f_r[1] -min(yp_r),p2f_r[1] -min(yp_r) ])
ax, plt = pEXP.plot_field(XFs,YFs,UF, shape,
            Vminmax=[0,0.02])
ax.plot(coords_liner_s[2:5,0],coords_liner_s[2:5,1],'k')
ax.plot(p1_s,p2_s,'r')

x_axis = 'x'
# xx, yy, distance, profile, ax, plt = pEXP.plot_line(Xs, Ys, U ,p1_s,p2_s,
#                                                     interp=True, smooth=True,
#                                                     xaxis = x_axis)
p1_s = np.array([p1f_r[0] -min(xp_r)+offset,p1f_r[1] -min(yp_r)])
p2_s = np.array([p2f_r[0] -min(xp_r)+offset,p2f_r[1] -min(yp_r) +
             abs(p1f_r[1] -min(yp_r) - (yA_r[0]+yA_r[1])/2)
             -abs(p2f_r[1] -min(yp_r) - (yA_r[0]+yA_r[1])/2)])
# p1_s = np.array([p1[0] -min(xp_r)+260,0])
# p2_s = np.array([p2[0] -min(xp_r)+260,800])
xx, yy, distance, profile, ax,plt = pEXP.plot_line(XFs, YFs, UF ,p1_s,p2_s,
                                        interp=False,
                                        x_resolution = interp_size,
                                        smooth=smooth,
                                        xaxis = x_axis,
                                        Vminmax=[0,0.2],
                                        limx=[100,650],
                                        limy=[100,650],
                                        showfig=True)
plt.savefig('profile_field.png', dpi=450)

# xA_r_new = [p1_s[0]+xA_r[0]-xA_r[1], p1_s[0]-xA_r[0]+xA_r[1]]

#%% ------------------------------- plot publi mirror

ax, plt = pEXP.plot_field(XFs,YFs,UF, shape,Vminmax=[0,0.009])
ax.plot(coords_liner_s[2:5,0],coords_liner_s[2:5,1],'k')
plt.axis('square')
# plt.xlim(min(Xs),max(Ys))
# plt.ylim(min(Xs),max(Ys))
plt.xlim(300,500)
plt.ylim(300,500)
plt.savefig('publi_mirror_field.png', dpi=450)

#%% SAVE DATA AND PARAMETERS

dict_real_data = { "XYU" : [XFs,YFs,UF, UFs],
            "prl" : 60,
            "shape": shape,
            "coords_liner": coords_liner_s,
            "p12": [p1_s,p2_s]}

afile = open('fig6_data_m' + '.pkl', 'wb')
pickle.dump(dict_real_data, afile)
afile.close()

interp_size
interp
smooth
x_axis
nlay

#%% ------------------------------- Pad the edges of grids

# xp,yp,U, shape = dEXP.pad_edges(xp,yp,U,shape,pad_type=0) # reflexion=5
# pEXP.plot_line(xp, yp,U,p1,p2, interp=interp)

#%% ------------------------------- Plot the derivatives

xderiv = transform.derivx(XFs, YFs, UF, shape,order=0)
yderiv = transform.derivy(XFs, YFs, UF, shape,order=0)
zderiv = transform.derivz(XFs, YFs, UF, shape,order=0)

# interp = True
pEXP.plot_line(XFs, YFs, xderiv ,p1_s,p2_s,title='xderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('xderiv' + str(file) + '.png', dpi=450)

pEXP.plot_line(XFs, YFs, yderiv ,p1_s,p2_s,title='yderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('yderiv' + str(file) + '.png', dpi=450)

pEXP.plot_line(XFs, YFs, zderiv ,p1_s,p2_s,title='zderiv',x_resolution= interp_size,
            savefig=False, interp=interp, smooth=smooth,  Xaxis=x_axis)

plt.savefig('zderiv' + str(file) + '.png', dpi=450)

#%% ------- upward continuation of the field data
p = [p1_s,p2_s]

mesh, label_prop = dEXP.upwc(XFs, YFs, zp, UF, shape,
              zmin=0, zmax=max_elevation, nlayers=nlay,
              qorder=qorder)

plt, cmap = pEXP.plot_xy(mesh, label=label_prop,Xaxis=x_axis,
                     Vminmax=[0,0.0125], p1p2=p)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
plt.savefig('upwc voltage' + str(file) + '.png', dpi=450)

#%% DEXP ratio

# x_axis = 'x'
qratio = [1,0]
mesh_dexp, label_dexp = dEXP.dEXP_ratio(XFs, YFs, zp, UF, shape,
              zmin=0, zmax=max_elevation, nlayers=nlay,
              qorders=qratio)
fig = plt.figure(figsize=(6, 2), dpi=450)
ax = plt.gca()
plt, cmap = pEXP.plot_xy(mesh_dexp, label=label_dexp,
          markerMax=True,qratio=str(qratio),Vminmax=[0,0.075],
          p1p2=np.array([p1_s,p2_s]), ax=ax, Xaxis=x_axis) #, ldg=)
# plt, cmap = pEXP.plot_xy(mesh_dexp, label=label_dexp,
#              markerMax=True,qratio=str(qratio)
#              ax=ax, Xaxis=x_axis) #, ldg=)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('ratio\nvoltage (V)')

# if x_axis=='y':
# square([xA_r_new[0], xA_r_new[1], -z1, -z2])
# else:
# square([yA_r[0], yA_r[1], -z1, -z2])
plt.xlim([200,600])
plt.savefig('fig3b.png', dpi=450)
plt.savefig('fig3b.pdf', dpi=450)
plt.savefig('fig3b.svg', dpi=450)
plt.savefig('fig3b.eps', dpi=450)
# plt.legend('')

# %% ridges identification

dEXP.ridges_minmax_plot(XFs, YFs, mesh, p1_s, p2_s,
                                  label=label_prop,
                                  interp=interp,x_resolution= interp_size,
                                  smooth=smooth,
                                  fix_peak_nb=2,
                                  method_peak='find_peaks',
                                  showfig=True,
                                  Xaxis=x_axis)
#%%
# or  find_peaks or peakdet or spline_roots
# dfI,dfII, dfIII = dEXP.ridges_minmax(Xs, Ys, mesh, p1_s, p2_s,interp=interp,x_resolution= interp_size,
#                                       label=label_prop,fix_peak_nb=2,
#                                       smooth=smooth, # true = low pass, otherwise specify the filter to apply
#                                       method_peak='find_peaks',
#                                       showfig=True,
#                                       Xaxis=x_axis)

D = dEXP.ridges_minmax(XFs, YFs, mesh, p1_s, p2_s,
                                  label=label_prop,
                                  method_peak='find_peaks',
                                  fix_peak_nb=2,
                                  returnAmp=True,
                                  showfig=True,
                                  Xaxis=x_axis,
                                  interp=interp,x_resolution= interp_size,
                                  smooth = smooth,
                                  qorder=qorder)

dfI, dfII, dfIII =  D[0:3]
hI, hII, hIII  = D[3:6]
heights  = D[3:6]

#%% ------------------------------- plot ridges over continuated section

fig = plt.figure(figsize=(6, 2), dpi=450)
ax = plt.gca()
plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax, Xaxis=x_axis,
        Vminmax=[0,0.0125], p1p2=p)
cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc\nvoltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax)
plt.xlim([200,600])
ax.set_xlabel('y (m)')
ax.set_ylabel('elevation\n(m)')
plt.savefig('fig3a.png', dpi=450)
plt.savefig('fig3a.pdf', dpi=450)
plt.savefig('fig3a.svg', dpi=450)
plt.savefig('fig3a.eps', dpi=450)
# plt.savefig('ridges_raw_field.png', dpi=450)

#%% ------------------------------- filter ridges regionally constrainsted)

dfI_f,dfII_f, dfIII_f = dEXP.filter_ridges(dfI,dfII,dfIII,
                                        minDepth=minAlt_ridge,
                                        maxDepth=maxAlt_ridge,
                                        minlength=5,rmvNaN=True,
                                        xmin=100, xmax=700,
                                        Xaxis=x_axis)

df_f = dfI_f, dfII_f, dfIII_f

#%% ------------------------------- plot ridges fitted over continuated section

fig, ax1 = plt.subplots(figsize=(15,3))

plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax, Xaxis=x_axis,
          Vminmax=[0,0.0125], p1p2=p)
pEXP.plot_ridges_harmonic(dfI_f,dfII_f,dfIII_f,ax=ax,label=True)

df_fit = dEXP.fit_ridges(df_f, rmvOutliers=True) # fit ridges on filtered data

pEXP.plot_ridges_sources(df_fit, ax=ax, z_max_source=-max_elevation*2,
                      ridge_type=[0,1,2],ridge_nb=None)

cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax)
plt.xlim([200,600])

plt.savefig('ridgesfield.png', dpi=450)
# square([y1, y2, z1, z2])
#%%

fig, ax1 = plt.subplots(figsize=(15,3))

plt, cmap = pEXP.plot_xy(mesh, label=label_prop, ax=ax1, Xaxis=x_axis,
          Vminmax=[0,0.0125], p1p2=p)
pEXP.plot_ridges_harmonic(dfI_f,dfII_f,dfIII_f,ax=ax1,label=True)

df_fit = dEXP.fit_ridges(df_f, rmvOutliers=False) # fit ridges on filtered data

ax2 = pEXP.plot_ridges_sources(df_fit, ax=ax1, z_max_source=-max_elevation,
                      ridge_type=[0,1,2],ridge_nb=None)

labels_ax = ax.get_yticks()
labels_ax= labels_ax[labels_ax>0]

labels_ax2 = ax2.get_yticks()
labels_ax2= labels_ax2[labels_ax2<0]

ax.set_yticks(labels_ax)
ax2.set_yticks(labels_ax2)

# Adjust the plotting range of two y axes
org1 = 0.0  # Origin of first axis
org2 = 0.0  # Origin of second axis
pos = 0.2  # Position the two origins are aligned
align.yaxes(ax1, org1, ax2, org2, pos)

cbar = plt.colorbar(cmap,shrink=0.25, pad=0.04)
cbar.set_label('upwc voltage (V)')
plt.tight_layout()
pEXP.plot_ridges_harmonic(dfI,dfII,dfIII,ax=ax1)
plt.xlim([200,600])

plt.savefig('ridgesfield.png', dpi=450)