from tapiriik.services.service_base import ServiceAuthenticationType, ServiceBase
from tapiriik.services.service_record import ServiceRecord
from tapiriik.services.api import APIException, UserException, UserExceptionType
from tapiriik.services.interchange import UploadedActivity, ActivityType, ActivityStatistic, ActivityStatisticUnit, WaypointType, Waypoint, Location, Lap
from tapiriik.settings import WEB_ROOT, MAPMYFITNESS_CLIENT_KEY, MAPMYFITNESS_CLIENT_SECRET

from datetime import datetime, timedelta
from urllib.parse import parse_qs, urlencode
import json
import pytz
import requests
from django.core.urlresolvers import reverse
from requests_oauthlib import OAuth1

import logging

logger = logging.getLogger(__name__)

class MapMyFitnessService(ServiceBase):
    ID = "mapmyfitness"
    DisplayName = "MapMyFitness"
    DisplayAbbreviation = "MMR"
    AuthenticationType = ServiceAuthenticationType.OAuth
    UserAuthorizationURL = None
    AuthenticationNoFrame = True
    OutstandingOAuthRequestTokens = {}
    IsNew = True

    _activityMappings = {"16": ActivityType.Running,
                         "11": ActivityType.Cycling,
                         "41": ActivityType.MountainBiking,
                         "9": ActivityType.Walking,
                         "24": ActivityType.Hiking,
                         "398": ActivityType.DownhillSkiing,
                         "397": ActivityType.CrossCountrySkiing,  # actually "backcountry" :S
                         "107": ActivityType.Snowboarding,
                         "86": ActivityType.Skating,  # ice skating
                         "15": ActivityType.Swimming,
                         "57": ActivityType.Rowing,  # canoe/rowing
                         "211": ActivityType.Elliptical,
                         "21": ActivityType.Other}
    SupportedActivities = list(_activityMappings.values())

    def WebInit(self):
        redirect_uri = WEB_ROOT + reverse("oauth_return", kwargs={"service": "mapmyfitness"})
        params = {'client_id': MAPMYFITNESS_CLIENT_KEY,
                  'response_type': 'code',
                  'redirect_uri': redirect_uri}
        self.UserAuthorizationURL = \
            "https://api.mapmyfitness.com/v7.1/oauth2/authorize/?" + urlencode(params)

    def GenerateUserAuthorizationURL(self, session, level=None):
        oauth = OAuth1(MAPMYFITNESS_CLIENT_KEY, client_secret=MAPMYFITNESS_CLIENT_SECRET)
        response = requests.post("https://api.mapmyfitness.com/v7.1/oauth2/request_token", auth=oauth)
        credentials = parse_qs(response.text)
        token = credentials["oauth_token"][0]
        self.OutstandingOAuthRequestTokens[token] = credentials["oauth_token_secret"][0]
        reqObj = {"oauth_token": token, "oauth_callback": WEB_ROOT + reverse("oauth_return", kwargs={"service": "mapmyfitness"})}
        return "https://api.mapmyfitness.com/v7.1/oauth2/authorize?" + urlencode(reqObj)

    def _apiHeaders(self, serviceRecord):
        return {"Authorization": "Bearer " + serviceRecord.Authorization["Token"],
                "Accept-Charset": "UTF-8"}

    def _getUserId(self, serviceRecord):
        response = requests.get("https://api.mapmyfitness.com/v7.1/user/self", headers=self._apiHeaders(serviceRecord))
        responseData = response.json()
        return responseData["id"]

    def RetrieveAuthorizationToken(self, req, level):
        from tapiriik.services import Service

        code = req.GET.get("code")
        params = {"grant_type": "authorization_code",
                  "code": code,
                  "client_id": MAPMYFITNESS_CLIENT_KEY,
                  "client_secret": MAPMYFITNESS_CLIENT_SECRET,
                  "redirect_uri": WEB_ROOT + reverse("oauth_return", kwargs={"service": "mapmyfitness"})}

        response = requests.post("https://api.mapmyfitness.com/v7.1/oauth2/access_token",
                                 data=urlencode(params),
                                 headers={"Content-Type": "application/x-www-form-urlencoded",
                                          "api-key": MAPMYFITNESS_CLIENT_KEY})

        if response.status_code != 200:
            raise APIException("Invalid code")
        token = response.json()["access_token"]

        uid = self._getUserId(ServiceRecord({"Authorization": {"Token": token}}))

        return (uid, {"Token": token})

    def RevokeAuthorization(self, serviceRecord):
        # there doesn't seem to be a way to revoke the token
        pass

    def _getActivityTypeHierarchy(self, headers):
        if hasattr(self, "_activityTypes"):
            return self._activityTypes
        response = requests.get("https://api.mapmyfitness.com/v7.1/activity_type", headers=headers)
        data = response.json()
        self._activityTypes = {}
        for actType in data["_embedded"]["activity_types"]:
            self._activityTypes[actType["_links"]["self"][0]["id"]] = actType
        return self._activityTypes

    def _resolveActivityType(self, actType, headers):
        self._getActivityTypeHierarchy(headers)
        if actType in self._activityMappings:
            return self._activityMappings[actType]
        activity = self._activityTypes[actType]
        parentLink = activity["_links"].get("parent")
        if parentLink is not None:
            parentId = parentLink[0]["id"]
            return self._resolveActivityType(parentId, headers)
        return ActivityType.Other

    def DownloadActivityList(self, serviceRecord, exhaustive=False):
        logger.debug("DownloadActivityList")
        allItems = []
        headers=self._apiHeaders(serviceRecord)
        nextRequest = '/v7.1/workout/?user=' + str(serviceRecord.ExternalID)
        while True:
            response = requests.get("https://api.mapmyfitness.com" + nextRequest, headers=headers)
            if response.status_code != 200:
                if response.status_code == 401 or response.status_code == 403:
                    raise APIException("No authorization to retrieve activity list", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))
                raise APIException("Unable to retrieve activity list " + str(response), serviceRecord)
            data = response.json()
            allItems += data["_embedded"]["workouts"]
            nextLink = data["_links"].get("next")
            if not exhaustive or not nextLink:
                break
            nextRequest = nextLink[0]["href"]

        activities = []
        exclusions = []
        for act in allItems:
            # TODO catch exception and add to exclusions
            activity = UploadedActivity()
            activityID = act["_links"]["self"][0]["id"]
            activity.StartTime = datetime.strptime(act["start_datetime"], "%Y-%m-%dT%H:%M:%S%z")
            activity.Notes = act["notes"] if "notes" in act else None

            # aggregate
            aggregates = act["aggregates"]
            elapsed_time_total = aggregates["elapsed_time_total"] if "elapsed_time_total" in aggregates else "0"
            activity.EndTime = activity.StartTime + timedelta(0, round(float(elapsed_time_total)))
            activity.Stats.TimerTime  = ActivityStatistic(ActivityStatisticUnit.Seconds, value=float(elapsed_time_total))
            activity.Stats.MovingTime = ActivityStatistic(ActivityStatisticUnit.Seconds, value=float(elapsed_time_total))
            if "active_time_total" in aggregates:
                activity.Stats.MovingTime = ActivityStatistic(ActivityStatisticUnit.Seconds, value=float(aggregates["active_time_total"]))

            if "distance_total" in aggregates:
                activity.Stats.Distance = ActivityStatistic(ActivityStatisticUnit.Meters, value=float(aggregates["distance_total"]))

            if "speed_min" in aggregates:
                activity.Stats.Speed.Min = float(aggregates["speed_min"])
            if "speed_max" in aggregates:
                activity.Stats.Speed.Max = float(aggregates["speed_max"])
            if "speed_avg" in aggregates:
                activity.Stats.Speed.Average = float(aggregates["speed_avg"])

            if "heartrate_min" in aggregates:
                activity.Stats.HR.update(ActivityStatistic(ActivityStatisticUnit.BeatsPerMinute, min=float(aggregates["heartrate_min"])))
            if "heartrate_max" in aggregates:
                activity.Stats.HR.update(ActivityStatistic(ActivityStatisticUnit.BeatsPerMinute, max=float(aggregates["heartrate_max"])))
            if "heartrate_avg" in aggregates:
                activity.Stats.HR = ActivityStatistic(ActivityStatisticUnit.BeatsPerMinute, avg=float(aggregates["heartrate_avg"]))

            if "cadence_min" in aggregates:
                activity.Stats.Cadence.update(ActivityStatistic(ActivityStatisticUnit.RevolutionsPerMinute, min=int(aggregates["cadence_min"])))
            if "cadence_max" in aggregates:
                activity.Stats.Cadence.update(ActivityStatistic(ActivityStatisticUnit.RevolutionsPerMinute, max=int(aggregates["cadence_max"])))
            if "cadence_avg" in aggregates:
                activity.Stats.Cadence = ActivityStatistic(ActivityStatisticUnit.RevolutionsPerMinute, avg=int(aggregates["cadence_avg"]))

            if "power_min" in aggregates:
                activity.Stats.Power.update(ActivityStatistic(ActivityStatisticUnit.Watts, min=int(aggregates["power_min"])))
            if "power_max" in aggregates:
                activity.Stats.Power.update(ActivityStatistic(ActivityStatisticUnit.Watts, max=int(aggregates["power_max"])))
            if "power_avg" in aggregates:
                activity.Stats.Power = ActivityStatistic(ActivityStatisticUnit.Watts, avg=int(aggregates["power_avg"]))

            activityTypeLink = act["_links"].get("activity_type")
            activityTypeID = activityTypeLink[0]["id"] if activityTypeLink is not None else None

            privacyLink = act["_links"].get("privacy")
            privacyID = privacyLink[0]["id"] if privacyLink is not None else None
            activity.Private = privacyID == "0"

            activity.Type = self._resolveActivityType(activityTypeID, headers)

            activity.ServiceData = {
                "ActivityID": activityID,
                "activityTypeID": activityTypeID,
                "privacyID": privacyID
                }
            activity.CalculateUID()
            activities.append(activity)
        return activities, exclusions

    def DownloadActivity(self, serviceRecord, activity):
        activityID = activity.ServiceData["ActivityID"]
        logger.debug("DownloadActivity %s" % activityID)

        lap = Lap(stats=activity.Stats, startTime=activity.StartTime, endTime=activity.EndTime)
        activity.Laps = [lap]
        lap.Waypoints = []

        response = requests.get("https://api.mapmyfitness.com/v7.1/workout/" + activityID + "/?field_set=time_series", headers=self._apiHeaders(serviceRecord))
        data = response.json()

        activity.GPS = False
        activity.Stationary = True

        # add waypoints to laps
        if "time_series" in data and "position" in data["time_series"]:
            activity.Stationary = False
            for pt in data["time_series"]["position"]:
                timestamp = pt[0]
                wp = Waypoint(activity.StartTime + timedelta(seconds=round(timestamp)))

                pos = pt[1]
                if ("lat" in pos and "lng" in pos) or "elevation" in pos:
                    wp.Location = Location()
                    if "lat" in pos and "lng" in pos:
                        wp.Location.Latitude = pos["lat"]
                        wp.Location.Longitude = pos["lng"]
                        activity.GPS = True
                    if "elevation" in pos:
                        wp.Location.Altitude = pos["elevation"]

                lap.Waypoints.append(wp)

        return activity

    def UploadActivity(self, serviceRecord, activity):

        if activity.Private:
            privacy_option_id = "0"
        else:
            privacy_option_id = "3"

        activity_type_id = [k for k,v in self._activityMappings.items() if v == activity.Type][0]
        if not activity_type_id:
            activity_type_id = "1"

        elapsed_time_total = activity.EndTime - activity.StartTime

        aggregates = {
            "elapsed_time_total": elapsed_time_total.seconds
        }

        if activity.Stats.Distance.Value is not None:
            aggregates["distance_total"] = activity.Stats.Distance.asUnits(ActivityStatisticUnit.Meters).Value

        if activity.Stats.TimerTime.Value is not None:
            aggregates["active_time_total"] = activity.Stats.TimerTime.asUnits(ActivityStatisticUnit.Seconds).Value
        elif activity.Stats.MovingTime.Value is not None:
            aggregates["active_time_total"] = activity.Stats.MovingTime.asUnits(ActivityStatisticUnit.Seconds).Value
        else:
            aggregates["active_time_total"] = (activity.EndTime - activity.StartTime).total_seconds()

        speed_stats = activity.Stats.Speed.asUnits(ActivityStatisticUnit.MetersPerSecond)
        if speed_stats.Average is not None:
            aggregates["speed_avg"] = speed_stats.Average
        if speed_stats.Min is not None:
            aggregates["speed_min"] = speed_stats.Min
        if speed_stats.Max is not None:
            aggregates["speed_max"] = speed_stats.Max

        hr_stats = activity.Stats.HR.asUnits(ActivityStatisticUnit.BeatsPerMinute)
        if hr_stats.Average is not None:
            aggregates["heart_rate_avg"] = hr_stats.Average
        if hr_stats.Min is not None:
            aggregates["heart_rate_max"] = hr_stats.Min
        if hr_stats.Max is not None:
            aggregates["heart_rate_max"] = hr_stats.Max

        if activity.Stats.Power.Average is not None:
            aggregates["power_avg"] = activity.Stats.Power.asUnits(ActivityStatisticUnit.Watts).Average
        if activity.Stats.Power.Min is not None:
            aggregates["power_min"] = activity.Stats.Power.asUnits(ActivityStatisticUnit.Watts).Min
        if activity.Stats.Power.Max is not None:
            aggregates["power_max"] = activity.Stats.Power.asUnits(ActivityStatisticUnit.Watts).Max

        if activity.Stats.Cadence.Average is not None:
            aggregates["cadence_avg"] = activity.Stats.Cadence.asUnits(ActivityStatisticUnit.RevolutionsPerMinute).Average
        elif activity.Stats.RunCadence.Average is not None:
            aggregates["cadence_avg"] = activity.Stats.RunCadence.asUnits(ActivityStatisticUnit.StepsPerMinute).Average

        if activity.Stats.Cadence.Min is not None:
            aggregates["cadence_min"] = activity.Stats.Cadence.asUnits(ActivityStatisticUnit.RevolutionsPerMinute).Min
        elif activity.Stats.RunCadence.Min is not None:
            aggregates["cadence_min"] = activity.Stats.RunCadence.asUnits(ActivityStatisticUnit.StepsPerMinute).Min

        if activity.Stats.Cadence.Max is not None:
            aggregates["cadence_max"] = activity.Stats.Cadence.asUnits(ActivityStatisticUnit.RevolutionsPerMinute).Max
        elif activity.Stats.RunCadence.Max is not None:
            aggregates["cadence_max"] = activity.Stats.RunCadence.asUnits(ActivityStatisticUnit.StepsPerMinute).Max

        # time series
        position = []
        heartrate = []
        power = []
        distance = []
        speed = []
        cadence = []
        for wp in activity.GetFlatWaypoints():
            time = wp.Timestamp - activity.StartTime
            time = time.seconds
            if wp.Location:
                pos = {}
                if wp.Location.Latitude is not None and wp.Location.Longitude is not None:
                    pos["lat"] = wp.Location.Latitude
                    pos["lng"] = wp.Location.Longitude
                if wp.Location.Altitude is not None:
                    pos["elevation"] = wp.Location.Altitude
                pt = [time, pos]
                position.append(pt)
            if wp.HR is not None:
                pt = [time, round(wp.HR)]
                heartrate.append(pt)
            if wp.Power is not None:
                pt = [time, round(wp.Power)]
                power.append(pt)
            if wp.Distance is not None:
                pt = [time, round(wp.Distance)]
                distance.append(pt)
            if wp.Speed is not None:
                pt = [time, round(wp.Speed)]
                speed.append(pt)
            if wp.Cadence is not None:
                pt = [time, round(wp.Cadence)]
                cadence.append(pt)
            elif wp.RunCadence is not None:
                pt = [time, round(wp.RunCadence)]
                cadence.append(pt)

        time_series = {}
        if position:
            time_series["position"] = position
        if heartrate:
            time_series["heartrate"] = heartrate
        if power:
            time_series["power"] = power
        if distance:
            time_series["distance"] = distance
        if speed:
            time_series["speed"] = speed
        if cadence:
            time_series["cadence"] = cadence

        upload_data = {
            "start_datetime": activity.StartTime.isoformat(),
            "start_locale_timezone": activity.TZ.zone,
            "name": activity.Name,
            "privacy": "/v7.1/privacy_option/%s/" % privacy_option_id,
            "activity_type": "/v7.1/activity_type/%s/" % activity_type_id,
            "aggregates": aggregates,
            "time_series": time_series
        }

        if activity.Notes:
            upload_data["notes"] = activity.Notes

        upload_resp = requests.post(
            "https://api.mapmyfitness.com/v7.1/workout/",
             headers=self._apiHeaders(serviceRecord),
             data=json.dumps(upload_data))
        if upload_resp.status_code != 201:
            raise APIException("Could not upload activity %s %s" % (upload_resp.status_code, upload_resp.text))

        return upload_resp.json()["_links"]["self"][0]["id"]

    def DeleteCachedData(self, serviceRecord):
        pass

    def DeleteActivity(self, serviceRecord, uploadId):
        pass


#!/usr/bin/env python

import rospy
import numpy as np
from std_msgs.msg import Float32

class ArmJointStateManager:
    def __init__(self):
        self.base_yaw_angle = np.radians(0.0)
        self.base_pitch_angle = np.radians(0.0)
        self.jack_triangle_base_angle = np.radians(0.0)
        self.__link_length = 0.200

        self.basejoint_names = ['joint_arm_yaw', 'joint_arm_pitch']

        self.jackjoint_names_left = ['virtualJoint_armLinear_base_left',
                                     'joint_armLinear_left_A_1', 'joint_armLinear_left_B_1',
                                     'joint_armLinear_left_A_2', 'joint_armLinear_left_B_2',
                                     'joint_armLinear_left_A_3', 'joint_armLinear_left_B_3']

        self.jackjoint_names_right = ['virtualJoint_armLinear_base_right',
                                      'joint_armLinear_right_A_1', 'joint_armLinear_right_B_1',
                                      'joint_armLinear_right_A_2', 'joint_armLinear_right_B_2',
                                      'joint_armLinear_right_A_3', 'joint_armLinear_right_B_3']

    def __get_pos_jacks_arr(self, jackjoint_names, sig):
        ret_arr = range(len(jackjoint_names))
        for idx, name in enumerate(jackjoint_names):
            if idx == 0:  # the 1st joint is slider
                ret_arr[idx] = sig * (self.__link_length / 2.0) * (1.0 - np.cos(self.jack_triangle_base_angle / 2.0))
            elif idx == 1:  # the 2nd joint is half joint
                ret_arr[idx] = -sig * self.jack_triangle_base_angle / 2.0
            elif idx % 2 == 1:
                ret_arr[idx] = -sig * self.jack_triangle_base_angle
            elif idx % 2 == 0:
                ret_arr[idx] = sig * self.jack_triangle_base_angle
            else:
                ret_arr[idx] = 0.0
        return ret_arr

    def __get_pos_jacks_arr_left(self):
        return self.__get_pos_jacks_arr(self.jackjoint_names_left, 1.0)

    def __get_pos_jacks_arr_right(self):
        return self.__get_pos_jacks_arr(self.jackjoint_names_right, -1.0)

    def __armjack_callback(self, top_angle):
        self.jack_triangle_base_angle = np.radians((360.0 - 2.0 * float(top_angle.data)) / 2.0)

    def __armbase_yaw_callback(self, yaw_angle):
        self.base_yaw_angle = np.radians(yaw_angle.data)

    def __armbase_pitch_callback(self, pitch_angle):
        self.base_pitch_angle = np.radians(pitch_angle.data)

    def get_jointstate_name(self):
        return self.basejoint_names + self.jackjoint_names_left + self.jackjoint_names_right

    def get_jointstate_position(self):
        return [self.base_yaw_angle,
                self.base_pitch_angle] + self.__get_pos_jacks_arr_left() + self.__get_pos_jacks_arr_right()

    def run(self):
        rospy.Subscriber("armjack_triangle_topangle_deg", Float32, self.__armjack_callback)
        rospy.Subscriber("armbase_yaw_deg", Float32, self.__armbase_yaw_callback)
        rospy.Subscriber("armbase_pitch_deg", Float32, self.__armbase_pitch_callback)


from ..internals.cache.redis import delete_keys
from ..internals.database.database import get_cursor, get_conn, return_conn
from shutil import rmtree
from os.path import join
import config

def delete_post_cache_keys(service, artist_id, post_id):
    keys = [
        'post:' + service + ':' + str(artist_id) + ':' + str(post_id)
    ]

    delete_keys(keys)

def delete_all_post_cache_keys():
    keys = ['all_post_keys']

    delete_keys(keys)

def post_exists(service, artist_id, post_id):
    cursor = get_cursor()
    cursor.execute("SELECT * FROM posts WHERE id = %s AND \"user\" = %s AND service = %s", (post_id, artist_id, service,))
    return len(cursor.fetchall()) > 0

def remove_post_if_flagged_for_reimport(service, user, post):
    conn = get_conn()
    cursor = get_cursor()

    cursor.execute('SELECT * FROM booru_flags WHERE service = %s AND "user" = %s AND id = %s', (service, user, post))
    existing_flags = cursor.fetchall()
    if len(existing_flags) == 0:
        return

    cursor.execute('DELETE FROM booru_flags WHERE service = %s AND "user" = %s AND id = %s', (service, user, post))
    cursor.execute('DELETE FROM posts WHERE service = %s AND "user" = %s AND id = %s', (service, user, post))
    conn.commit()
    return_conn(conn)
    rmtree(join(
        config.download_path,
        'attachments',
        '' if service == 'patreon' else service,
        user,
        post
    ), ignore_errors=True)
    rmtree(join(
        config.download_path,
        'files',
        '' if service == 'patreon' else service,
        user,
        post
    ), ignore_errors=True)


import requests
import sys
import csv
import re
from bs4 import BeautifulSoup
docterNameArray=[]
i=1
# fp=open("Plastic_Surgery_India.csv","w")
# url="https://find.plasticsurgery.org/country/india/?page=1"
fp=open("Plastic_Surgery_UAE.csv","w")

csvwriter = csv.writer(fp)
csvwriter.writerow(['Firstname','Lastname','degree','address','pincode','telephone'])
# print(len(data))
p = re.compile(r'^(\s+)?(Mr(\.)?|Mrs(\.)?)?(?P<FIRST_NAME>.+)(\s+)(?P<LAST_NAME>.+)$', re.IGNORECASE)

k=0
for k in range(0,5):

	url="https://find.plasticsurgery.org/?q=United%2BArab%2BAmirati&page=1"
	# print(url)
	result=requests.get(url)
	src = result.content
	soup = BeautifulSoup(src, "lxml")
	data=soup.find_all("div",{"class":"resultSet col-md-6 col-xs-12"})
	def getPlasticSurgeryData(data):
		for city in data:
			value=city.find_all("div",{"class":"col-xs-12"})
			for content in value:
				docterName=content.find("span",{"itemprop":"givenName"}).text
				address=content.find("p",{"itemprop":"address"}).text
				address=' '.join(address.split())
				telephone=re.sub('[^A-Za-z0-9]+', '',content.find("p",{"itemprop":"telephone"}).text)
				if(len(telephone)==12):
					telephone=telephone[2:len(telephone)]
				if(len(telephone)==13):
					telephone=telephone[2:len(telephone)]
				if(len(telephone)==14):
					telephone=telephone[2:len(telephone)]

				data=docterName.split(",")
				m = p.match(data[0])
				if(m != None):
					first_name = m.group('FIRST_NAME')
					last_name = m.group('LAST_NAME')
				degree=data[1]

				if(address!=None):
					zipcode = re.search(r'\d{6}(?:-\d{4})?(?=\D*$)', address)
					if(zipcode!=None):
						pincode=zipcode.group()
					else:
						pincode="-"

				if(len(telephone)>8):
					print(telephone)
					telephone='+'+telephone

				elif(len(telephone)<=8):
					telephone='+971'+telephone

				if(docterName not in docterNameArray):
					docterNameArray.append(docterName)
					csvwriter = csv.writer(fp)
					csvwriter.writerow([first_name,last_name,degree,address,pincode,telephone])

	getPlasticSurgeryData(data)
	i=1
	nextPageData=soup.find("div",{"class":"col-xs-12 results-pager"})
	while(nextPageData.find("a",{"rel":"next"}).has_attr('href')):
		i=i+1
		url="https://find.plasticsurgery.org/?q=United%2BArab%2BAmirati&page="+str(i)
		# print(url)
		result=requests.get(url)
		src = result.content
		soup = BeautifulSoup(src, "lxml")
		data=soup.find_all("div",{"class":"resultSet col-md-6 col-xs-12"})
		getPlasticSurgeryData(data)
		nextPageData=soup.find("div",{"class":"col-xs-12 results-pager"})

print(len(docterNameArray))


#!/usr/bin/env python
'''
Condicionales [Python]
Ejercicios de práctica
---------------------------
Autor: Inove Coding School
Version: 1.1

Descripcion:
Programa creado para que practiquen los conocimietos adquiridos durante la semana
'''

__author__ = "Inove Coding School"
__email__ = "alumnos@inove.com.ar"
__version__ = "1.1"

def ej1():
    # Ejercicios de práctica con números
    print("Ingrese dos numeros")
    numero_1 = float(input())
    numero_2 = float(input())

    resta = numero_1 - numero_2

    if resta > 0:
      print("El restulado es positivo")
    elif resta < 0:
      print("El resultado es negativo")
    else:
      print("El resultado es igual a cero")

    '''
    Realice un programa que solicite por consola 2 números
    Calcule la diferencia entre ellos e informe por pantalla
    si el resultado es positivo, negativo o cero.

    '''

def ej2():
# Ejercicios de práctica con números

    print("Ingrese tres numero")
    numero_1 = int(input())
    numero_2 = int(input())
    numero_3 = int(input())

    division = numero_1 % 2
    if division == 0:
      print("numero par")
    else:
      print("numero impar")

    division = numero_2 % 2
    if division == 0:
      print("numero par")
    else:
      print("numero impar")

    division = numero_3 % 2
    if division == 0:
      print("numero par")
    else:
      print("numero impar")

    #Realice un programa que solicite el ingreso de tres números
    #enteros, y luego en cada caso informe si el número es par
    #o impar.
    #Para cada caso imprimir el resultado en pantalla.

def ej3():
    # Ejercicios de práctica con números

    numero_1 = float(input("Ingrese el primer numero: \n"))
    numero_2 = float(input("ingrese el segundo numero: \n"))

    print("ingrese la operacion que desea realizar")

    operacion = input()
    suma = numero_1 + numero_2
    resta = numero_1 - numero_2
    multiplicacion = numero_1 * numero_2
    division = numero_1 / numero_2
    potencia = numero_1 ** numero_2

    if operacion == "+":
      print("la suma es",suma)
    elif operacion == "-":
      print("la resta es:",resta)
    elif operacion == "*":
      print("la multiplicacion es:",multiplicacion)
    elif operacion == "/":
      print("la division es:",division)
    elif operacion == "**":
      print(numero_1,"elevado a",numero_2,"es:",potencia)
    else:
      print("operacion invalida")

    '''
    Realice una calculadora, se ingresará por línea de comando dos números
    Luego se ingresará como tercera entrada al programa el símbolo de la operación
    que se desea ejecutar
    - Suma (+)
    - Resta (-)
    - Multiplicación (*)
    - División (/)
    - Exponente/Potencia (**)

    Se debe efectuar el cálculo correcto según la operación ingresada por consola
    Imprimir en pantalla la operación realizada y el resultado

    '''

def ej4():
    # Ejercicios de práctica con cadenas

    print("ingrese tres palabras")
    palabra_1 = str(input())
    palabra_2 = str(input())
    palabra_3 = str(input())

    print("ingrese la operacion que quiere realizar")
    numero = int(input())

    if numero == 1:
      if (palabra_1 < palabra_2 < palabra_3):
        print(palabra_3,palabra_2,palabra_1)
      elif (palabra_1 < palabra_3 < palabra_2):
        print(palabra_2,palabra_3,palabra_1)
      elif (palabra_2 < palabra_1 < palabra_3):
        print(palabra_3,palabra_1,palabra_2)
      elif (palabra_2 < palabra_3 < palabra_1):
        print(palabra_1,palabra_3,palabra_2)
      elif (palabra_3 < palabra_1 < palabra_2):
        print(palabra_2,palabra_1,palabra_3)
      elif (palabra_3 < palabra_2 < palabra_1):
        print(palabra_1,palabra_2,palabra_3)

    if numero == 2:
      if len(palabra_1) <= len(palabra_2) <= len(palabra_3):
        print(palabra_3,palabra_2,palabra_1)
      elif len(palabra_1) <= len(palabra_3) <= len(palabra_2):
        print(palabra_2,palabra_3,palabra_1)
      elif len(palabra_2) <= len(palabra_1) <= len(palabra_3):
        print(palabra_3,palabra_1,palabra_2)
      elif len(palabra_2) <= len(palabra_3) <= len(palabra_1):
        print(palabra_1,palabra_3,palabra_2)
      elif len(palabra_3) <= len(palabra_1) <= len(palabra_2):
        print(palabra_2,palabra_1,palabra_3)
      elif len(palabra_3) <= len(palabra_2) <= len(palabra_1):
        print(palabra_1,palabra_2,palabra_3)

    '''
    Realice un programa que solicite por consola 3 palabras cualesquiera
    Luego el programa debe consultar al usuario como quiere ordenar las palabras
    1 - Ordenar por orden alfabético (usando el operador ">")
    2 - Ordenar por cantidad de letras (longitud de la palabra)

    Si se ingresa "1" por consola se deben ordenar las 3 palabras por orden alfabético
    e imprimir en pantalla de la mayor a la menor

    Si se ingresa "2" por consola se deben ordenar las 3 palabras por cantidad de letras
    e imprimir en pantalla de la mayor a la menor

  '''

def ej5():
    # Ejercicios de práctica con números

    print("Igrese el valor de las temperaturas de los ultimos tres dias")

    temperatura_1 = float(input())
    temperatura_2 = float(input())
    temperatura_3 = float(input())

    maximo = max(temperatura_1,temperatura_2,temperatura_3)
    promedio = (temperatura_1 + temperatura_2 + temperatura_3) / 3

    print("la temperatura mayor es",maximo)
    print("la temperatura menor es",min(temperatura_1,temperatura_2,temperatura_3))
    print("la temperatura promedio es",promedio)

    # Inove: Perfecto que hayas aprovechado las funciones de "max" y "min" para
    # el cálculo, esas y otras funciones las veremos la clase que viene
    # junto con los métodos de bucles o loops, y listas.

    '''
    Realice un programa que solicite ingresar tres valores de temperatura
    De las temperaturas ingresadas por consola determinar:
    1 - ¿Cuáles de ellas es la máxima temperatura ingresada?
    2 - ¿Cuáles de ellas es la mínima temperatura ingresada?
    3 - ¿Cuál es el promedio de las temperaturas ingresadas?

    En cada caso imprimir en pantalla el resultado

    '''

if __name__ == '__main__':
    #print("Ejercicios de práctica")
    #ej1()
    #ej2()
    #ej3()
    #ej4()
    ej5()


from dtk.utils.analyzers.BaseAnalyzer import BaseAnalyzer
from simtools.AnalyzeManager.AnalyzeManager import AnalyzeManager
from simtools.SetupParser import SetupParser

class TestAnalyzer(BaseAnalyzer):
    def __init__(self, sim_keys):
        super(TestAnalyzer, self).__init__()
        self.sim_keys = sim_keys
        self.sim_data = {}
        # self.run_number = None
        # self.x_temp = None
        # self.mp_start = None
    def filter(self, sim_metadata):
        self.sim_data[sim_metadata['sim_id']] = {x: sim_metadata[x] for x in self.sim_keys}

    def apply(self, parser):
        pass

if __name__ == "__main__":
    SetupParser.init('HPC')

    sim_ids = ['ccf42399-a377-e711-9401-f0921c16849d', 'cbf42399-a377-e711-9401-f0921c16849d']
    sweep_vars = ['Run_Number', 'x_Temporary_Larval_Habitat', 'add_health_seeking.start_day']

    for sim_id in sim_ids:
        am = AnalyzeManager(verbose=False)
        analyzer = TestAnalyzer(sim_id, sweep_vars)
        am.add_experiment('b9e12809-9277-e711-9401-f0921c16849d')
        am.add_analyzer(analyzer)
        am.analyze()
        print(analyzer.sim_data)


''' This program will use a set of existing messages stored in a database to develop a ML model
    based on how the messages are classified. Then the  model will be stored in a pickle file
    that should be specified in the arguments.

    Parameters:
    argument 1: Database file name
    arugment 2: Pickle file name

    Returns:
    GridSearchCV: trained model

'''

import sys
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
import re
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
import pickle
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')

def load_data(database_filepath):
    ''' Loads data from the database and split messages and categoreis

    Parameters:
        database_filepath: The file path of the database to load dataframe

    Returns:
        X: dataframe of the messages
        Y: dataframe of the categories
        category_names: the names of the categories columns
    '''

    #Create an engine and load database
    engine = create_engine('sqlite:///{}'.format(database_filepath))
    df = pd.read_sql_table("MessagesTable2",engine)

    #X is the messages, Y is the categories
    X = df['message']
    Y = df.drop(['id','message','original','genre'],axis=1)
    category_names = Y.columns

    return X,Y,category_names

def tokenize(text):
    ''' takes a text  and normalize,lemmatize, and tokenize it.

    Parameters:
        text: the text string

    Returns:
        tokens: List of the tokenized lemmatize and normalized
    '''

    text = re.sub(r'[^a-zA-Z0-9]',' ',text.lower())
    stop_words = stopwords.words("english")
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(word).strip() for word in tokens if word not in stop_words]
    return tokens

def build_model():
    ''' this function builds the pipeline and gridsearchcv for the model

    Returns:
        cv: GridSearchCV using pipeline and parameters
    '''
    #build the pipeline
    pipeline = Pipeline([
        ('vect', CountVectorizer(tokenizer=tokenize)),
        ('tfidf',TfidfTransformer()),
        ('clf',MultiOutputClassifier(RandomForestClassifier()))
    ])

    # I tried multiple parameters for the GridSearchCV
    # to shorten the time some of those are commented out since GridSearch took forever to finish
    parameters = {
        'vect__max_df':(0.75,1),
#        'vect__max_features':(None,2000),
#         'vect__ngram_range': ((1, 1), (1, 2)),
#         'tfidf__use_idf': (True),
#         'tfidf__norm': ('l1', 'l2'),
#         'clf__n_jobs':(1,2),
#         'clf__estimator__n_estimators': [50, 100],
#         'clf__estimator__n_estimators': [75, 100],
#         'clf__estimator__min_samples_split': [2, 4],
#         'clf__estimator__criterion':('gini', 'entropy'),
    }

    cv = GridSearchCV(pipeline, param_grid=parameters)
    return cv

# method to convert the predected array to dataframe
def convert_to_df(Y_pred,columns):
    ''' convert an array to dataframe

    Parameters:
        Y_pred: predicted values in array format
        columns: names of the columns

    Returns:
        df: reutns a dataframe constructed from the predicted array
    '''
    df = pd.DataFrame(Y_pred)
    df.columns = columns
    return df

def evaluate_model(model, X_test, Y_test, category_names):
    '''evaluate the model and pring out the resutls of the classificaiton

    Parameters:
        model: the trained model
        X_test: the messages to classify
        Y_test: the actual results of the classificaiton
        category_names: the names of the categories
    '''
    Y_pred = model.predict(X_test)
    Y_pred_df = convert_to_df(Y_pred,category_names)
    for column in Y_test.columns:
        names= [column+"-0",column+"-1",column+"-2"]
        print(classification_report(np.array(Y_test[column]), np.array(Y_pred_df[column]),target_names=names))

def save_model(model, model_filepath):
    '''Save the model inot a pickle file
    Parametesr:
        model: The model to save
        model_filepath: the path to save the pickle file in and the name of the file
    '''
    pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


import numpy
import matplotlib.pyplot as plt
plt.switch_backend('agg')

import os

import clawpack.geoclaw.units as units
import clawpack.geoclaw.surge.storm as storm

from load_synthetic_storms import load_other_chaz_storms
from load_synthetic_storms import load_chaz_storms

from netCDF4 import Dataset

def ncdump(nc_fid, verb=True):
    '''
    ncdump outputs dimensions, variables and their attribute information.
    The information is similar to that of NCAR's ncdump utility.
    ncdump requires a valid instance of Dataset.

    Parameters
    ----------
    nc_fid : netCDF4.Dataset
        A netCDF4 dateset object
    verb : Boolean
        whether or not nc_attrs, nc_dims, and nc_vars are printed

    Returns
    -------
    nc_attrs : list
        A Python list of the NetCDF file global attributes
    nc_dims : list
        A Python list of the NetCDF file dimensions
    nc_vars : list
        A Python list of the NetCDF file variables
    '''
    def print_ncattr(key):
        """
        Prints the NetCDF file attributes for a given key

        Parameters
        ----------
        key : unicode
            a valid netCDF4.Dataset.variables key
        """
        try:
            print("\t\ttype:", repr(nc_fid.variables[key].dtype))
            for ncattr in nc_fid.variables[key].ncattrs():
                print('\t\t%s:' % ncattr,\
                      repr(nc_fid.variables[key].getncattr(ncattr)))
        except KeyError:
            print("\t\tWARNING: %s does not contain variable attributes" % key)

    # NetCDF global attributes
    nc_attrs = nc_fid.ncattrs()
    if verb:
        print("NetCDF Global Attributes:")
        for nc_attr in nc_attrs:
            print('\t%s:' % nc_attr, repr(nc_fid.getncattr(nc_attr)))
    nc_dims = [dim for dim in nc_fid.dimensions]  # list of nc dimensions
    # Dimension shape information.
    if verb:
        print("NetCDF dimension information:")
        for dim in nc_dims:
            print("\tName:", dim)
            print("\t\tsize:", len(nc_fid.dimensions[dim]))
            print_ncattr(dim)
    # Variable information.
    nc_vars = [var for var in nc_fid.variables]  # list of nc variables
    if verb:
        print("NetCDF variable information:")
        for var in nc_vars:
            if var not in nc_dims:
                print('\tName:', var)
                print("\t\tdimensions:", nc_fid.variables[var].dimensions)
                print("\t\tsize:", nc_fid.variables[var].size)
                print_ncattr(var)
    return nc_attrs, nc_dims, nc_vars

#nc_fid_chaz_ny_nc = Dataset("../data/synthetic-storm-data/LongIsland_chaz_test.nc", 'r')
#nc_attrs, nc_dims, nc_vars = ncdump(nc_fid_chaz_ny_nc)

#obs = [["Obs_ibtracks_knaff15_Mumbai.nc", "Obs NC", "other_chaz", "k", 2016-1879+1]]
data = [["LongIsland_chaz_test.nc", "Obs NC", "other_chaz", "k", 2016-1879+1]]

data_path = os.path.join("../data", "synthetic-storm-data", data[0][0])
storms = load_chaz_storms(path = data_path,
                mask_distance = None,
                mask_coordinate = None,
                mask_category = 4,
                categorization = "NHC")

test_storm = storms[0]

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2)

title_font = {'fontname':'Arial', 'size':'12', 'color':'black', 'weight':'normal',
          'verticalalignment':'bottom'} # Bottom vertical alignment for more space
axis_font = {'fontname':'Arial', 'size':'12'}

axes = fig.add_subplot(2, 2, 1)
axes.plot(test_storm.max_wind_speed, test_storm.max_wind_radius, 'ro', label="Knaff Max Wind Radius")
axes.set_xlabel('Max Wind Speed (knots)', **axis_font)
axes.set_ylabel('Max Wind Radius (nmi)', **axis_font)
axes.legend()

axes = fig.add_subplot(2, 2, 2)
axes.plot(units.convert(test_storm.max_wind_speed, 'knots', 'm/s'),
units.convert(test_storm.max_wind_radius, 'nmi', 'km'), 'ro-', label="Knaff MWR")
axes.set_xlabel('Max Wind Speed (m/s)', **axis_font)
axes.set_ylabel('Max Wind Radius (km)', **axis_font)
axes.legend()

axes = fig.add_subplot(2, 2, 3)
axes.plot(range(0, test_storm.max_wind_speed.shape[0]),
test_storm.max_wind_speed, 'bo-', label="Max Wind Speed")
axes.set_xlabel('Obs Count', **axis_font)
axes.set_ylabel('Max Wind Speed (knots)', **axis_font)
axes.legend()

axes = fig.add_subplot(2, 2, 4)
axes.plot(range(0, test_storm.max_wind_radius.shape[0]),
test_storm.max_wind_radius, 'ro-', label="Knaff MWR")
axes.set_xlabel('Obs Count', **axis_font)
axes.set_ylabel('Max Wind Radius (nmi)', **axis_font)
axes.legend()

plt.yticks(fontsize=10)
plt.xticks(fontsize=10)
plt.tight_layout()
plt.savefig('MaxWindRadiusCheck.pdf')


import random
class KnightsTour():

    def __init__(self, board_axis):
        if len(board_axis) != 2:
            raise Exception('Error Baord Axis incorrect', board_axis)
        self.board_axis = board_axis
        self.m = board_axis[0]
        self.n = board_axis[1]
        self.board = None
        self.moves = {}

    def build_board(self):

        if self.board is None:
            self.board_hash = {}
            self.available_moves = {}
            self.board = " "
            self.board += "_" * 5 * self.m
            self.board += "\n"
            for y in range(1, self.m + 1):
                self.board += "|____"
                for x in range(1, self.n):
                    if self.board_hash.get((x,y)):
                        self.board += "|_xx_"
                    else:
                        self.board += "|____"

                self.board += "|\n"
        return self.board_hash
    def print_board(self):
        print(self.board)
    def run(self):
        board = self.build_board()
        possible_moves = self.get_all_available_moves()
        first_move = random.sample(self.available_moves.keys(), 1)[0]
        self.find_tour(first_move, 1)
        import pdb; pdb.set_trace()

    def get_all_available_moves(self):
        number_of_moves = self.m * self.n
        for y in range(1, self.m + 1):
            for x in range(1, self.n + 1):
                coordinates = (x, y)
                self.board_hash[coordinates] = False
                self.available_moves[coordinates] = self.find_all_moves(x,y)

    def find_all_moves(self, x, y):
        ## gets all valid moves for coordinates
        moves = [
            (x + 1, y + 2),
            (x + 1, y - 2),
            (x - 1, y + 2),
            (x - 1, y - 2),

            (x + 2, y + 1),
            (x + 2, y - 1),
            (x - 2, y + 1),
            (x - 2, y - 1),
        ]

        # less than 'n' and 'm' because we want to make sure its less than
        # the edges
        return [m for m in moves if(
                            m[0] > 0
                            and m[0] <= self.m
                            and m[1] <= self.n
                            and m[1] > 0
                            and not self.moves.get(m))]

    def find_tour(self, coordinates, i):
        moves = self.available_moves.get(coordinates)

        if len(moves) > 0:
            self.moves[coordinates] = i
            self.board_hash[coordinates] = True
            found_move = moves[0]
            self.get_all_available_moves()
            self.find_tour(found_move, i + 1)
        else:
            self.fix_euler(i)
        return self.moves

    def fix_euler(self, i):
        new_moves = self.find_max_moves()
        coords = new_moves[0]
        new_move = random.sample(new_moves[1], 1)[0]
        index = self.moves[coords]
        temp_moves = self.moves.copy()
        self.moves = {}
        self.moves[new_move] = index
        all_moves = [mv for mv in temp_moves.keys()]
        for y in range(0, len(temp_moves.keys())):
            new_i = index - 1
            if new_i > 1:
                new_c = all_moves[new_i]
                self.moves[new_c] = new_i
            else:
                another_temp = temp_moves.copy()
                for k,v in temp_moves.items():
                    another_temp[k] += 1
                self.moves.update(another_temp)

    def find_max_moves(self):
        moves = [m for m in self.moves.keys()]
        temp_array = []
        for move in moves:
            available_moves = self.find_all_moves(move[0], move[1])
            if len(available_moves) > 0:
                temp_array.append([move, available_moves])

        return random.sample(temp_array, 1)[0]

k = KnightsTour((5,5)).run()


import pickle as pk
import cv2
import dlib
from align import AlignDlib
from load_data import load_metadata
import numpy as np
from model import create_model
import time

def load_image(path):
    img = cv2.imread(path, 1)
    # OpenCV loads images with color channels
    # in BGR order. So we need to reverse them
    return img[...,::-1]

def dbEncodings_to_Vect(database):
    db_vect = []
    names = []
    for (name,db_enc) in database:
        names.append(name)
        db_vect.append(db_enc)
    db_vect = np.array(db_vect).T       # db_vect.shape = (128, len(db))
    return names, db_vect

#Initializing camera input
camera_port = 0
camera = cv2.VideoCapture(camera_port)

# Open database
database = pk.load(open("database.pkl", "rb"))
# Make encoding vectors
names, db_vect = dbEncodings_to_Vect(database)

# Initialize the OpenFace face alignment utility
alignment = AlignDlib('models/landmarks.dat')

#Create NN model
model = create_model()
model.load_weights('weights/nn4.small2.v1.h5')

# New time record :  0.41 on Asus A555L without GPU acceleration
bb = None # Setting the bounding_box as None

while True:
    start_time = time.time()

    # Reading image from webcam
    retval, image = camera.read()
    image = image[...,::-1] #Now in RGB
    image = cv2.resize(image, (300, 300))
    temp_image = image

    try :

        # 1. IMAGE --> EMBEDDING
        # Turn and crop the face
        image, bb = alignment.align(96, image, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) #image.shape = (96, 96, 3)
        #image = misc.imresize(image, (96, 96))

        # Scale RGB values to interval [0,1]
        image = (image / 255.).astype(np.float32)
        # Obtain the embedding vector for image
        encoded_image = model.predict(np.expand_dims(image, axis=0))[0]   # Time taken = 0.04

        # 2.EMBEDDING --> DISTANCES
        encoded_image = np.reshape(encoded_image, (128,1))
        # Computing Dictances
        dist_vect = np.sum(np.square(np.subtract(db_vect, encoded_image)), axis=0, dtype=np.float32, keepdims=True)
        dist_vect = np.squeeze(dist_vect)

        # 3.DISTANCES --> NAME
        result = dist_vect.min()
        if(result < 0.3):
            predict_name = names [ list(dist_vect).index(result) ]
            print(predict_name, result)
        else:
            predict_name = "Unknown"

    except:
        print ("Face not detected !")

    #Draw the bounding box
    if (bb is not None):
        cv2.rectangle(temp_image, (bb.left(), bb.top()) ,(bb.right(), bb.bottom()), (0,255,0),1)
        cv2.putText(temp_image, predict_name+" : "+str(result)+"%", (bb.left(),bb.top()-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0))

    #Display the image with bounding box
    temp_image = temp_image[...,::-1] #Now in BGR
    cv2.imshow("Window", temp_image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

    print("Time elapsed : ", time.time() - start_time)

camera.release()
cv2.destroyAllWindows()


#!/usr/bin/python
import sys
fout=open("out.csv","a")
# first file:
for line in open("1.csv"):
    fout.write(line)
# now the rest:
for num in range(2,int(sys.argv[1])):
    f = open(str(num)+".csv")
    for line in f:
         fout.write(line)
    f.close() # not really needed
fout.close()


#!/usr/bin/env python3
import asyncio
from datetime import datetime
import sys
import pandas as pd
import numpy as np
import boto3
from decimal import Decimal
from dateutil.relativedelta import relativedelta
from cbpro import PublicClient
import math
import time
import matplotlib.pyplot as plt
from copra.websocket import Channel, Client
from copra.rest import Client as CopraClient

def flatten(l): return [item for sublist in l for item in sublist]

def plot_multi(data, cols=None, spacing=.1, **kwargs):

    from pandas import plotting

    # Get default color style from pandas - can be changed to any other color list
    if cols is None:
        cols = data.columns
    if len(cols) == 0:
        return
    colors = getattr(getattr(plotting, '_matplotlib').style,
                     '_get_standard_colors')(num_colors=len(cols))

    # First axis
    ax = data.loc[:, cols[0]].plot(label=cols[0], color=colors[0], **kwargs)
    ax.set_ylabel(ylabel=cols[0])
    lines, labels = ax.get_legend_handles_labels()

    for n in range(1, len(cols)):
        # If you need multiple y-axes
        # ax_new = ax.twinx()
        # ax.spines['right'].set_position(('axes', 1 + spacing * (n - 1)))

        # Plot using the existing y-axis
        data.loc[:, cols[n]].plot(
            ax=ax, label=cols[n], color=colors[n % len(colors)])
        ax.set_ylabel(ylabel=cols[n])

        # Proper legend position
        line, label = ax.get_legend_handles_labels()
        lines += line
        labels += label

    ax.legend(lines, labels, loc=0)
    return ax

class Data:
    def __init__(self, n):
        self.public_client = PublicClient()
        self.end_time = datetime.now()
        self.increment = relativedelta(minutes=200)
        self.start_time = self.end_time - self.increment
        self.period = n
        self.runs = 2 * int(math.ceil(n / 200))

    def get_data(self):
        data = []
        while self.runs > 0:
            response = self.public_client.get_product_historic_rates(
                'BTC-USD', start=self.start_time, end=(self.end_time), granularity=60)
            if 'message' in response:
                self.runs += 1
                time.sleep(0.5)
            else:
                data.append(response)
                self.start_time -= self.increment
                self.end_time -= self.increment
                self.runs -= 1
                time.sleep(0.5)
        return (flatten(data))

    def get_ewm(self):
        data = self.get_data()
        labels = ['timestamp', 'low', 'high', 'open', 'close', 'volume']
        df = pd.DataFrame.from_records(data, columns=labels)
        df = df.set_index('timestamp')
        df = df.sort_index(ascending=True)
        df['ewm'] = df['close'].ewm(
            span=self.period, min_periods=0, adjust=False, ignore_na=False).mean()
        # plt.figure()
        # df.plot()
        # plot_multi(df, cols=["close", "ewm"])
        return df['ewm'].iloc[-1]

class EWM:
    def __init__(self, average, period):
        self.average = average
        self.period = period
        self.multiplier = 2.0 / float(1.0 + period)

    def next(self, value):
        self.average = ((value - self.average) *
                        self.multiplier) + self.average

    def get(self):
        return self.average

class Tick:
    def __init__(self, tick_dict, ewm):
        self.sequence = tick_dict['sequence']
        self.product_id = tick_dict['product_id']
        self.best_bid = float(tick_dict['best_bid'])
        self.best_ask = float(tick_dict['best_ask'])
        self.price = float(tick_dict['price'])
        self.side = tick_dict['side']
        self.size = float(tick_dict['last_size'])
        self.time = datetime.strptime(
            tick_dict['time'], '%Y-%m-%dT%H:%M:%S.%fZ')
        self.ewm = ewm

    def get_price(self):
        return self.price

    @property
    def spread(self):
        return self.best_ask - self.best_bid

    def __repr__(self):
        rep = "{}\t\t\t\t {}\n".format(self.product_id, self.time)
        rep += "======================================================================================\n"
        rep += "Price: ${:.2f}\t\tSize: {:.8f}\tSide: {: >5}\tewm: {: >5}\n".format(
            self.price, self.size, self.side, self.ewm)
        rep += "Best ask: ${:.2f}\tBest bid: ${:.2f}\tSpread: ${:.2f}\tSequence: {}\n".format(
            self.best_ask, self.best_bid, self.spread, self.sequence)
        rep += "======================================================================================\n"
        return rep

class Ticker(Client):
    def __init__(self, ewm, loop, channels):
        super().__init__(loop=loop, channels=channels)
        self.ewm = ewm

    def on_message(self, message):
        if message['type'] == 'ticker' and 'time' in message:
            average = self.ewm.get()
            tick = Tick(message, average)
            price = tick.get_price()
            self.ewm.next(price)
            print(tick, "\n\n")
        if message['type'] == 'heartbeat':
            print(message)
        if message['type'] == 'level1':
            print(message)

class Trade:
    def __init__(self, bids, asks, ewm):
        self.bids = bids
        self.asks = asks
        self.ewm = ewm

"""
    def should_buy():
        if price < ewm & (price / ewm) < pct:
            return True
        else:
            return False

    def should_sell():
        if price > ewm & (price / ewm) > pct:
            return True
        else:
            return False
            """

period = 200
data = Data(period)
starter = data.get_ewm()
average = EWM(starter, period)

#plt.show()

"""

loop = asyncio.get_event_loop()
channels = Channel('heartbeat', 'BTC-USD')
ticker = Ticker(ewm=average, loop=loop, channels=channels)

"""
loop2 = asyncio.get_event_loop()

client = CopraClient(loop2)

async def get_order_book():
    book = await client.order_book(product_id='BTC-USD', level=1)
    book['time'] = datetime.now().isoformat()
    print(book)

while True:
    starttime=time.time()
    loop2.run_until_complete(get_order_book())
    time.sleep(1.0 - ((time.time() - starttime) % 1.0))

loop2.run_until_complete(get_order_book())

try:
    loop.run_forever()
except KeyboardInterrupt:
    loop.run_until_complete(ticker.close())
    loop.close()


import numpy as np
from tqdm.auto import tqdm

from globals import components_layer, nums_L0, datasets
from Sampler import Sampler

class SizeEstimatorSimulator(object):
    def __init__(self):
        pass

    @classmethod
    def get_error(cls, real_value, estimation):
        return 100 * abs(real_value - estimation) / real_value

    @classmethod
    def compute_size_for_dataset(cls,
                                 exp_id,
                                 dataset,
                                 l1_size_range,
                                 l0_method="greedy",
                                 l0_size=-1):
        print(f"Starting experiment #{exp_id}")
        title = dataset.title
        sampler = Sampler()
        sampler.init(title)
        sampler.generate_L0(l0_method, l0_size)
        high_subgraph = sampler.high_subgraph
        layer_manager = sampler.layer_manager
        size_estimator = sampler.size_estimator
        statistics = sampler.statistics
        ground_truth_avg_deg_down = statistics.avg_degrees()[1]

        layers = layer_manager.get_layers(components_layer[title])

        nodes_from_L1 = np.random.choice(high_subgraph.L1_list,
                                         min([len(high_subgraph.L1_list), max(l1_size_range)]),
                                         replace=False).tolist()
        L1_total_size = len(high_subgraph.L1_list)
        first_two_layers_size = len(high_subgraph.L0_list) + L1_total_size
        real_size = len(layers[-1]) + first_two_layers_size

        size_estimator.update_up(nodes_from_L1[:l1_size_range[0]])
        estimated_up_degs = [size_estimator.estimate_average_deg_up()]
        for i in tqdm(range(1, len(l1_size_range))):
            size_estimator.update_up(nodes_from_L1[l1_size_range[i-1]:l1_size_range[i]])
            estimated_up_degs.append(size_estimator.estimate_average_deg_up())

        sizes = [L1_total_size * deg_up / ground_truth_avg_deg_down for deg_up in estimated_up_degs]

        print("Done computing estimated sizes")
        return [cls.get_error(real_value=real_size, estimation=estimated_l2_size + first_two_layers_size) for estimated_l2_size in sizes]

    @classmethod
    def compute_size_for_dataset_l2_samples(cls,
                                            exp_id,
                                            dataset,
                                            l2_size_range,
                                            l0_method="greedy",
                                            l0_size=-1):
        print(f"Starting experiment #{exp_id}")
        title = dataset.title
        sampler = Sampler()
        sampler.init(title)
        sampler.generate_L0(l0_method, l0_size)
        high_subgraph = sampler.high_subgraph
        size_estimator = sampler.size_estimator
        statistics = sampler.statistics
        ground_truth_avg_deg_up = statistics.avg_degrees()[0]

        nodes, reachabilities = statistics.all_component_nodes_and_reachabilities()
        normed_reachabilities = np.array(reachabilities) / np.sum(reachabilities)

        nodes_from_comps = np.random.choice(nodes, max(l2_size_range), p=normed_reachabilities).tolist()

        first_two_layers_size = len(high_subgraph.L0_list) + len(high_subgraph.L1_list)
        real_size = len(nodes) + first_two_layers_size

        size_estimator.update_down(nodes_from_comps[:l2_size_range[0]])
        estimated_down_degs = [size_estimator.estimate_average_deg_down()]
        for i in tqdm(range(1, len(l2_size_range))):
            size_estimator.update_down(nodes_from_comps[l2_size_range[i - 1]:l2_size_range[i]])
            estimated_down_degs.append(size_estimator.estimate_average_deg_down())

        sizes = [len(high_subgraph.L1_list) * ground_truth_avg_deg_up / deg_down for deg_down in estimated_down_degs]

        print("Done computing estimated sizes")
        return [cls.get_error(real_value=real_size, estimation=estimated_l2_size + first_two_layers_size) for
                estimated_l2_size in sizes]

if __name__ == "__main__":
    x = SizeEstimatorSimulator.compute_size_for_dataset("Up", datasets[0], range(100, 1001, 100))
    print(x)
    x = SizeEstimatorSimulator.compute_size_for_dataset_l2_samples("Down", datasets[0], range(100, 1001, 100))
    print(x)


from django.test import TestCase
from django.test.client import Client
from persons.models import Person as P

class PersonTestCase(TestCase):
    def setUp(self):
        P.objects.create(short_name='mwon', full_name='Michael Won')
        P.objects.create(short_name='jim12', full_name='James D. Costning',
                                email='jim12@jimbo.people.we.com')
        P.objects.create(short_name='world_master')

    def test_persons_count(self):
        self.assertEqual(P.objects.count(), 3)

    def test_persons_fields(self):
        persons = P.objects.all()
        correct_values = {
            'mwon' : {'str' : 'mwon (Michael Won)', 'email' : None},
            'jim12' : {'str' : 'jim12 (James D. Costning)',
                                'email' : 'jim12@jimbo.people.we.com'},
            'world_master' : {'str' : 'world_master', 'email' : None},
        }

        for p in persons:
            vals = correct_values[p.short_name]
            self.assertEqual(str(p), vals['str'])
            self.assertEqual(p.email, vals['email'])


# -*- coding: utf-8 -*-
"""
Created on Wed Dec 13 16:50:28 2017

@author: Sian JM Brooke
"""
import numpy as np
from sklearn import preprocessing
from scipy.stats import boxcox, gamma

### Log Transformation ###
var2log = "Twitter Gender" #Col name to transform
logcol = "Twitter Gender Log" #name for new col

tgen = wgh
tgen[var2log] = tgen[var2log].dropna().reset_index(drop=True)
tgen = tgen[(tgen[var2log] != 0)]

#tgen[logcol] = np.log(tgen[var2log])
#https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.log.html

#tgen[logcol].hist()
#print("Skew before:",tgen[var2log].skew(), "Skew After:", tgen[logcol].skew())

varpro = "Twitter Gender"
tgen2 = wgh
tgen3 = tgen2[varpro].dropna().reset_index(drop=True)

### Preprocessing data from sklearn
centscale = preprocessing.scale(tgen3)
dt = pd.DataFrame(data=centscale)
#print(dt.hist())

tbxcx = preprocessing.scale(boxcox(tgen3+1)[0])
dt2 = pd.DataFrame(data=tbxcx)
#print(dt2.hist())
#bxcx = boxcox(tgen3)

bnry = ["Male", "Female"]
forbinry = wgh[wgh.Gender.isin(bnry)].reset_index(drop=True)

forr = forbinry[["Twitter Gender", "Twitter Hostility",
            "Twitter Help Offering", "Twitter Help Seeking"]].copy()
forr = forr.fillna(0)
### Preprocessing data from sklearn
def prepro(i):
    centscale = preprocessing.scale(forr[i])
    forr[i] = centscale
    return
prepro(list(forr))


# STDIN       Function
# -----       --------
# AABCAAADA   s = 'AABCAAADA'
# 3           k = 3

string = "AABCAAADA"
k = 3

for part in zip(*[iter(string)] * k):
    d = dict()
    print(''.join([ d.setdefault(c, c) for c in part if c not in d]))


#!/usr/bin/env python
############# MAIN ############
# Store strings into array.
list   = "The quick brown fox jumps over the lazy dog".split()

# Concatenate Strings
string = " ".join(list)

# Oputput Concatenated String
print("  Concatenation => " + string + ".")


from django.conf.urls import url

from . import views

app_name = "twitter"

urlpatterns = [
    url(regex=r"^$", view=views.HomeView.as_view(), name="home"),
    url(regex=r"^likes/$", view=views.FavoriteListView.as_view(), name="favorite_list"),
    url(
        regex=r"^(?P<screen_name>\w+)/$",
        view=views.UserDetailView.as_view(),
        name="user_detail",
    ),
    url(
        regex=r"^(?P<screen_name>\w+)/likes/$",
        view=views.AccountFavoriteListView.as_view(),
        name="account_favorite_list",
    ),
    url(
        regex=r"^(?P<screen_name>\w+)/(?P<twitter_id>\d+)/$",
        view=views.TweetDetailView.as_view(),
        name="tweet_detail",
    ),
]


import logging
import time as t

from setup import PROXY, TOKEN
from telegram.ext import CallbackContext, CommandHandler, Filters, MessageHandler, Updater, ConversationHandler, CallbackQueryHandler
from telegram import Bot, Update, InlineKeyboardMarkup, InlineKeyboardButton

from nltk.corpus import stopwords
from pymystem3 import Mystem
from string import punctuation
import json

mystem = Mystem()
russian_stopwords = stopwords.words("russian") + ["ко"]

bot = Bot(
        token=TOKEN,
        base_url=PROXY,  # delete it if connection via VPN
    )

def start(update: Update, context: CallbackContext):
    """Send a message when the command /start is issued."""
    text = f'Hello, {update.effective_user.first_name}! 👋\nWelcome to the smart bot that is able to detect and extract key words from text!\n Write /help to learn detailes.'
    update.message.reply_text(text)
    return text

def help(update: Update, context: CallbackContext):
    update.message.reply_text("In order to find key words in your document it is required that you send me a message containing document only. Make sure there are no other charachters. Preferred type of doc is .txt. Hope you'll enjoy!")

def get_file(update: Update, context: CallbackContext):
    print("get file function is started")
    overall_start = t.time()
    document = update["message"]["document"]["file_id"]
    file = context.bot.getFile(file_id=document)
    file.download("file.txt")
    key_words, tokens = keywords()
    overall_finish = t.time()
    update.message.reply_text(f"According to my findings, your file containes {len(key_words)} significant words.")
    update.message.reply_text(f"It took {overall_finish - overall_start} seconds.")
    update.message.reply_text(f"The length of your doc is {len(tokens)} words")
    update.message.reply_text(" ".join(key_words[:10]))

def keywords():
    print("key words is started")
    # making a list of all lemmas from the doc
    start = t.time()
    with open("file.txt", encoding="utf8") as doc:
        tokens = mystem.lemmatize(doc.read().lower())
        tokens = [token for token in tokens if token not in russian_stopwords \
                  and token != " " \
                  and token.strip() not in punctuation]
        tokens = [token for token in tokens if token.isalpha() and len(token) > 1]
    finish = t.time()
    print(f"the doc is lemmatized in {start - finish} secs")

    #opening the json with all the frequencies of words from NCRL and counting the length of the corpora
    start = t.time()
    with open("frequencies.json") as doc:
        corpora = json.load(doc)
        corp_total = 0
        for i in corpora:
            corp_total += corpora[i]
    finish = t.time()
    print(f"frequencies.json are open and ready to be used in {start - finish} secs")
    print(corpora["солнце"])

    start = t.time()
    doc_frqn = {}
    corp_frqn = {}

    for token in tokens:
        doc_frqn[token] = tokens.count(token) / len(tokens)
        try:
            corp_frqn[token] = corpora[token] / corp_total
        except KeyError:
            corp_frqn[token] = 0
    finish = t.time()
    print(f"numbers of entrances is calculated succesfully in {finish - start} secs")

    expected = {}
    for token in doc_frqn.keys():
        a = doc_frqn[token]
        b = corp_frqn[token]
        c = 1 - a
        d = 1 - b
        expected[token] = ((a + b) * (a + c)) / (a + b + c + d)

    print("expected frequency is calculated successfully ")

    chi = {}
    for token in expected.keys():
        o = doc_frqn[token]
        e = expected[token]
        chi[token] = ((o - e) ** 2) / e

    l = list(chi.items())
    l.sort(key=lambda i: i[1])
    l = l[::-1]
    print("chi2 value is calculated successfully")
    print(l)

    Q = 0.003932

    kw = []
    for i in l:
        if i[1] <= Q:
            kw.append(i[0])
    print("keywords list is formed")
    print("keywords function is finished")
    print(" ".join(kw))
    return(kw, tokens)

def main():
    bot = Bot(
        token=TOKEN,
        base_url=PROXY,  # delete it if connection via VPN
    )
    updater = Updater(bot=bot, use_context=True)
    updater.dispatcher.add_handler(MessageHandler(Filters.document, get_file))
    updater.dispatcher.add_handler(CommandHandler('start', start))
    updater.dispatcher.add_handler(CommandHandler('help', help))

    updater.start_polling()

    # Run the bot until you press Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()

if __name__ == '__main__':
    print("Bot is started")
    main()


import time
import tushare as ts
import pandas as pd
import datetime, calendar
import sys

#reload(sys)
#sys.setdefaultencoding('utf-8')

class StockBasics:
	def __init__(self, df):
		self.code = df.code
		self.name = df.name
		self.industry = df.industry
		self.area = df.area
		self.pe = df.pe
		self.outstanding = df.outstanding
		self.totals = df.totals
		self.totalAssets = df.totalAssets
		self.liquidAssets = df.liquidAssets
		self.fixedAssets = df.fixedAssets
		self.reserved = df.reserved
		self.reservedPerShare = df.reservedPerShare
		self.esp = df.esp
		self.bvps = df.bvps
		self.pb = df.pb
		self.timeToMarket = df.timeToMarket
		self.undp = df.undp
		self.perundp = df.perundp
		self.rev = df.rev
		self.profit = df.profit
		self.gpr = df.gpr
		self.npr = df.npr
		self.holders = df.holders

def update_stock_base_info():
#	df = ts.get_concept_classified()
#	df.to_csv('./data/stock_concept.csv')

#	df = ts.get_industry_classified()
#	df.to_csv('./data/stock_industry.csv')

	df = ts.get_stock_basics()
	df.to_csv('./data/stock_basics.csv')
	#print df
	return df

def update_k_data():
	df = pd.read_csv('./data/stock_basics.csv')
	for i in range(len(df.index)):
		stock0 =  StockBasics(df.ix[i])
		k_data = ts.get_k_data("%06d"%stock0.code)
		k_data.to_csv("./data/%06d.csv"%stock0.code)
		#print "./data/%06d.csv"%stock0.code

def is_stock_suspended(k_data):
	k_date_len = len(k_data.index)
	last_date = k_data.ix[k_date_len-1].date
	last_datetime = datetime.datetime.strptime(last_date, "%Y-%m-%d")
	deltaDays=datetime.timedelta(days=10)
	now = datetime.datetime.now()
	if (now - last_datetime) > deltaDays:
		return True
	else:
		return False

def is_rise_stock(k_data, percent,  days):
	if len(k_data.index) < days:
		return False
	tail_data = k_data.tail(days)
	if tail_data.iloc[0].close > tail_data.iloc[-1].close:
		return False
	nums = 0.0
	for i in range(len(tail_data.index)):
		if i == 0:
			continue
		if tail_data.iloc[i].close > tail_data.iloc[i-1].close:
			nums = nums + 1
	if nums/days*100 >= percent:
		rise_percent = tail_data.iloc[-1].close/tail_data.iloc[0].close - 1
		print ("%06d %f(%f/%f)(rise:%f|%f,%f)"%(tail_data.iloc[i].code, nums/days, nums, days, rise_percent, tail_data.iloc[0].close, tail_data.iloc[-1].close))

		return True
	else:
		return False

def is_3xx_stock(k_data):
	if k_data.ix[0].code > 299999 and k_data.ix[0].code < 400000:
		return True
	else:
		return False

def is_small_stock(socket_base, value):
	if socket_base.totals < value:
		return True
	else:
		return False

def is_last_day_rise_stock(k_data):
	if k_data.iloc[len(k_data.index)-1].close > k_data.iloc[len(k_data.index)-1].open:
		return True
	else:
		return False

def is_volume_abnormal_stock(k_data, multiplier, days):
	if len(k_data.index) < 10:
		return False

	if days > len(k_data.index):
		days = len(k_data.index)

	tail_data = k_data.tail(days)

	for i in range(days-1):
#		print ("-----")
#		print (tail_data.iloc[i].volume)
#		print (tail_data.iloc[i+1].volume)
		if (tail_data.iloc[i].volume*multiplier < tail_data.iloc[i+1].volume):
			return True

	return False

def is_new_stock(k_data, days):
	data_len = len(k_data.index)
	if data_len <= days and data_len > 0:
		print (data_len)
		print (k_data)
		return True
	return False

def stock_with_concept(concept):
	df = pd.read_csv('./data/stock_concept.csv')
	slist = []
	for i in range(len(df.index)):
		if df.iloc[i].c_name == concept:
			slist.append(df.iloc[i].code)
			#print ("%06d"%df.iloc[i].code)
	return slist

def update_data():
	update_stock_base_info()
	update_k_data()

def choose_stocks():
	df = pd.read_csv('./data/stock_basics.csv')
	for i in df.index:
		stock0 =  StockBasics(df.ix[i])
		k_data = pd.read_csv("./data/%06d.csv"%stock0.code)
		#is_new_stock(k_data, 5)
		#if not is_new_stock(k_data, 5):
		#	continue
		if not is_volume_abnormal_stock(k_data, 2, 2):
			continue
		is_rise_stock(k_data, 65, 20)

def find_stock_with_newstock():
	slist_tmp = stock_with_concept("次新股")
	slist = []
	for code in slist_tmp:
		k_data = pd.read_csv("./data/%06d.csv"%code)
#		for

def find_stock_with_volume():
	df = pd.read_csv('./data/stock_basics.csv')
	for i in df.index:
		stock0 =  StockBasics(df.ix[i])
		k_data = pd.read_csv("./data/%06d.csv"%stock0.code)
		if (is_volume_abnormal_stock(k_data, 10, 20)):
			print("%06d"%stock0.code)

def red_green_array(code):
	str = ""
	k_data = pd.read_csv("./data/%06d.csv"%code)
	for i in range(len(k_data.index)):
		if k_data.iloc[i].close >= k_data.iloc[i].open:
			str = str + '1'
		else:
			str = str + '0'

#	print (str)
	return str

def find_stock_with_rga(rga):
	df = pd.read_csv('./data/stock_basics.csv')
	for i in df.index:
		stock0 = StockBasics(df.ix[i])
		k_data = pd.read_csv("./data/%06d.csv"%stock0.code)
		if len(k_data.index) > 66:
			continue
		str = red_green_array(stock0.code)
		if rga in str:
			print ("%06d"%stock0.code)

def debug():
	find_stock_with_rga("011101010010010001010001111100011100001011100111101010000111")

def start():
#	print (slist)
	print("start...")
#	update_data()
	find_stock_with_rga("1111111000")
#	debug()
#	find_stock_with_volume()

start()
#debug()


# coding:utf-8
# python3
import random
L = [3, 44, 38, 5, 47, 15, 36, 26, 27, 2, 46, 4, 19, 50, 48]
random.shuffle(L)

def bubble_sort(a):
    n = len(a)
    for i in range(n):
        for j in range(1, n-i):
            if a[j-1] > a[j]:
                swap(a, j, j-1)
    return a

def select_sort(a):
    n = len(a)
    for i in range(n-1):
        minimum = i
        for j in range(i+1, n):
            if a[j] < a[minimum]:
                minimum = j
        swap(a, minimum, i)
    return a

def insert_sort(a):
    n = len(a)
    for i in range(1, n):
        t = a[i]
        j = i - 1
        while j >= 0 and t < a[j]:
            a[j+1] = a[j]
            j -= 1
        a[j+1] = t
    return a

def merge_sort(a):
    l = len(a)
    if l > 1:
        left = merge_sort(a[:(l//2)])
        right = merge_sort(a[(l//2):])
        a = []
        while len(left) != 0 and len(right) != 0:
            if left[0] < right[0]:
                a.append(left.pop(0))
            else:
                a.append(right.pop(0))
        if len(left) != 0:
            a.extend(left)
        elif len(right) != 0:
            a.extend(right)
    return a

def quick_sort(a):
    if len(a) > 1:
        pivot = a[0]
        rest = a[1:]
        smaller = [i for i in rest if i < pivot]
        larger = [j for j in rest if j >= pivot]
        return quick_sort(smaller) + [pivot] + quick_sort(larger)
    else:
        return a

def swap(a, x, y):
    a[x], a[y] = a[y], a[x]

if __name__ == '__main__':
    print(L)


#!/usr/bin/python3
"""Unittest for Base class
"""
import unittest
from models import base
from models.base import Base

class TestBase(unittest.TestCase):
    """ Class: TestBase : to test class Base
               functions must be inic by test_
               test class name could by named anyway
    """
    def test_doc(self):
        """ Function: test_doc
                      to test if have documentation
        """
        self.assertTrue(len(base.__doc__) > 0)
        self.assertTrue(len(Base.__doc__) > 0)
        self.assertTrue(len(Base.__init__.__doc__) > 0)

    def test_trivial(self):
        """ Function: test_trivial
                      to test trivial cases
        """
        b1 = Base()
        b2 = Base()
        b3 = Base()
        b4 = Base(12)
        b5 = Base()
        b6 = Base(-3)
        self.assertEqual(b1.id, 1)
        self.assertEqual(b2.id, 2)
        self.assertEqual(b3.id, 3)
        self.assertEqual(b4.id, 12)
        self.assertEqual(b5.id, 4)
        self.assertEqual(b6.id, -3)

if __name__ == '__main__':
    unittest.main()


from bs4 import BeautifulSoup
import requests
import re
import time
import random

def extractValue(car_property):
	return car_property.split(':')[1].strip("\"")

file_header="site,car_id,car_descrip,car_make,car_model,car_trim,car_year,car_category,car_mileage,car_price,car_extColor,car_transmission,car_driveTrain,car_vendor,car_location,car_detail_link,car_img_link"

carscomFileName = "carsdotcom_06272017_02_20001_Used"
site = "carsDotCom"
baseUrl ="https://www.cars.com"
carsFile = open("./"+carscomFileName+".csv",'w')
carsFile.write(file_header+"\n")

for pageNum in range(18,51):
	htmlSource="https://www.cars.com/for-sale/searchresults.action/?page="+str(pageNum)+"&perPage=100&rd=150&searchSource=UTILITY&sort=distance-nearest&stkTypId=28881&zc=20001"
	r = requests.get(htmlSource)
	data = r.text
	soup = BeautifulSoup(data,'html.parser')

	for listing in soup.find_all('div',class_ ="shop-srp-listings__listing"):
		car_id = str(listing['id'].split('-')[1])
		car_detail_link = str(baseUrl+listing.a['href'])
		car_img_link = str(listing.img['src'])
		car_descrip = str(listing.img['alt'])

		car_price = str(-1)
		price_line_entry = listing.find('span',class_='listing-row__price')
		if price_line_entry is not None :
			price_line = price_line_entry.contents
			if (len(price_line) >= 2) :
				car_price = str(price_line[1].replace(',',''))

		car_mileage = str(-1)
		mileage_line_entry = listing.find('span',class_='listing-row__mileage')
		if mileage_line_entry is not None :
			mileage_line = mileage_line_entry.contents
			mileage_list = mileage_line[0].split(':')
			if (len(mileage_list) == 2) :
				car_mileage = str(mileage_list[1].strip().replace(',',''))

		metainfo_items = listing.find_all('span',class_='listing-row__meta-item')
		#print(metainfo_items[0].strong.string.split(':')[0])

		car_extColor = ""
		car_transmission = ""
		car_driveTrain = ""
		if len(metainfo_items) == 3:
			car_extColor = str(metainfo_items[0].contents[1].strip())
			car_transmission = str(metainfo_items[1].contents[1].strip())
			car_driveTrain = str(metainfo_items[2].contents[1].strip())

		car_vendor = str((listing.find('a',class_='listing-row__dealer-name').contents)[0].strip())
		car_location = str(listing.find_all('span',class_="listing-row__distance")[1].contents[0].strip())

		det_list = listing.find('div',class_='listing-row__save').a['vehicle'].split(',')

		car_make = str(extractValue(det_list[2]))
		car_model = str(extractValue(det_list[4]))
		car_trim = str(extractValue(det_list[6]))
		car_year = str(extractValue(det_list[8]))
		car_category = str(extractValue(det_list[9]))

		carsFile.write(site+","+car_id+","+car_descrip+","+car_make+","+car_model+","+car_trim+","+car_year+","+car_category+","+car_mileage+","+car_price+","+car_extColor+","+car_transmission+","+car_driveTrain+","+car_vendor+","+car_location+","+car_detail_link+","+car_img_link+"\n")

	time.sleep(random.randint(45,70))

carsFile.close()


"""
INSTRUCION CONTINUE

Es la instrucion que interrumpe la ejecucion
de un unico ciclo

"""
print()
print("Inicion")

i = 0
while (i<10):
    i += 1
    if( i % 2 == 0):
        continue
    if(i>7):
        break
    print(i)
else:
    print("Else")

print("Fin")


#!/usr/bin/env python
import requests
import aiohttp
import asyncio
import dateutil.parser
import datetime
import pytz
import os.path
import functools
import sys
import json
from jinja2 import Template
from base64 import b64encode

COMMENT_URL = 'https://graph.facebook.com/v2.3/{object_id}/comments'

FB_COMMENT_LIMIT = 5000

COMMENT_LIMIT = 500

def get_comment_count(object_id, access_token):
    """Returns comment count for given object id"""

    comments_response = requests.get(
        COMMENT_URL.format(object_id=object_id),
        params=dict(access_token=access_token, summary=1))
    error = comments_response.json().get('error')
    if error:
        raise Exception(error['message'])
    total_count = comments_response.json()['summary']['total_count']
    return total_count

async def get_comments_chunk(session, object_id, access_token, limit, offset, comment_list):
    """
    Future for fetching single chunk of comment
    """
    after_token = b64encode(str(offset).encode('utf-8'))

    async with session.get(COMMENT_URL.format(object_id=object_id), params=dict(
            access_token=access_token,
            limit=limit,
            fields='comment_count,comments.limit(500){created_time,comment_count},created_time',
            after=after_token)) as response:
        response_json = await response.json()
        error = response_json.get('error')
        if error:
            print(error['message'])

        comment_list.extend(response_json['data'])
        for comment in response_json['data']:
            comment_list.extend(comment.get('comments', {}).get('data', []))

def fetch_data(object_id, access_token):
    """
    Asyncronously fetches first 5000 comments, other are fetched syncronously
    """

    comment_count = get_comment_count(object_id, access_token)
    loop = asyncio.get_event_loop()
    with aiohttp.ClientSession(loop=loop) as session:
        tasks = []
        comment_list = []
        session.max_offset_processed = 0

        for chunk_start in range(0, comment_count, COMMENT_LIMIT):
            tasks.append(asyncio.ensure_future(
                get_comments_chunk(session, object_id, access_token, COMMENT_LIMIT, chunk_start, comment_list)))
        loop.run_until_complete(asyncio.gather(*tasks))

    return comment_list

# Processing
def get_report_interval(min_date, max_date):
    """
    Returns best interval for gived min_date. If first comment was yesterday - it uses hours as delimiter,
    if three years ago and last today - years, if both posted three years ago in a same day - also hours
    """

    if max_date - min_date < datetime.timedelta(days=3):
        return 'hours'
    elif max_date - min_date < datetime.timedelta(days=30):
        return 'days'
    elif max_date - min_date < datetime.timedelta(days=365 * 4):
        return 'months'
    else:
        return 'years'

def strip_date_fields(date, interval):
    """Returns date of the period start, based on the interval"""

    interval_strips = {
        'hours': lambda: date.replace(minute=0, second=0),
        'days': lambda: date.replace(hour=0, minute=0, second=0),
        'months': lambda: date.replace(day=1, hour=0, minute=0, second=0),
        'years': lambda: date.replace(month=1, day=1, hour=0, minute=0, second=0),
    }
    return interval_strips[interval]()

def process_data(comment_list):
    """Converts list of comments to aggregated structure"""
    comment_dict = dict([(comment['id'], comment['created_time']) for comment in comment_list])
    dates = [dateutil.parser.parse(created_time) for created_time in comment_dict.values()]
    report_interval = get_report_interval(min(dates), max(dates))
    date_counters = {}

    for date in dates:
        stripped_date = strip_date_fields(date, report_interval)
        date_counters.setdefault(stripped_date, 0)
        date_counters[stripped_date] += 1
    return date_counters, report_interval

def repr_date(report_interval, date):
    """
    Converts date to presentation format, based on interval.
    For example, Wed, 18:00 for hour-based report or 2016 for year-based
    """
    reprs = {
        'hours': '%a, %H:00',
        'days': '%b %d',
        'months': '%b',
        'years': '%Y'
    }
    return date.strftime(reprs[report_interval])

def render_data(date_counters, report_interval):
    """ Renders processed data to template"""

    comment_labels = list(map(functools.partial(repr_date, report_interval), sorted(date_counters.keys())))
    comment_values = [date_counters[label] for label in sorted(date_counters.keys())]

    with open(os.path.dirname(os.path.abspath(__file__)) + '/report_template.j2') as template:
        template_html = Template(
            template.read()).render(
                comment_labels=json.dumps(comment_labels), comment_values=json.dumps(comment_values))

    with open(os.path.dirname(os.path.abspath(__file__)) + '/report.html', 'w') as report:
        report.write(template_html)

def main():
    #  Example: python fetcher.py 10153764743456749 CAAAABHLBW9ABAIjN3Rm9Y9WZCp0st3ZC81RNZAg...
    comment_list = fetch_data(sys.argv[1], sys.argv[2])
    date_counters, report_interval = process_data(comment_list)
    render_data(date_counters, report_interval)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Team 18

"""

###### importing necessary packages #####
import pandas as pd
from scipy.io import arff ### since the file is in arff format #####

###### dataset preprocessing ##########
data = arff.loadarff('OBS-Network-DataSet_2_Aug27.arff')
df = pd.DataFrame(data[0])
df.shape
df.describe()
####### Since the column are not named properly  so column renaming is required #########
columns_new=['Node','Utilised_Bandwith_Rate','Packet_Drop_Rate','Full_Bandwidth',
         'Average_Delay_Time_Per_Sec','Percentage_Of_Lost_Packet_Rate','Percentage_Of_Lost_Byte_Rate','Packet_Received_Rate',
         'of_Used_Bandwidth','Lost_Bandwidth','Packet_Size_Byte','Packet_Transmitted',
         'Packet_Received','Packet_lost','Transmitted_Byte','Received_Byte',
         '10_Run_AVG_Drop_Rate','10_Run_AVG_Bandwith_Use','10_Run_Delay','Node_Status',
         'Flood_Status','Class']
len(columns_new)
df.columns=columns_new
df.columns
df.info
df.describe()
################################
###### checking for missing values #######
missing_data = pd.DataFrame({'total_missing': df.isnull().sum(), 'perc_missing': (df.isnull().sum()/1075)*100})
missing_data
######### removing rows having null entries ######
df = df[pd.notnull(df['Packet_lost'])]

######## cross checking after removing null values ######

missing_data = pd.DataFrame({'total_missing': df.isnull().sum(), 'perc_missing': (df.isnull().sum()/1075)*100})
missing_data
df.shape
df.columns
############ Plotting  HIstogram #################
hist_se=df.hist(bins=10, figsize=(20, 15),grid=False)
###### from histogram we can see that the feature 'Node' and packet size byte are of no use ###
######### droping column which are of no use #######
df1=df.drop(columns=['Packet_Size_Byte','Node'])
df1.shape

df1.head
######### interchanging feature 'Flood_Status' and 'Node_Status' #######
df=df1.reindex(['Utilised_Bandwith_Rate', 'Packet_Drop_Rate', 'Full_Bandwidth',
       'Average_Delay_Time_Per_Sec', 'Percentage_Of_Lost_Packet_Rate',
       'Percentage_Of_Lost_Byte_Rate', 'Packet_Received_Rate',
       'of_Used_Bandwidth', 'Lost_Bandwidth', 'Packet_Transmitted',
       'Packet_Received', 'Packet_lost', 'Transmitted_Byte', 'Received_Byte',
       '10_Run_AVG_Drop_Rate', '10_Run_AVG_Bandwith_Use', '10_Run_Delay',
       'Flood_Status', 'Node_Status', 'Class'],axis=1)

df.shape
df.head(5)
######## Converting column values of 'Node_status' and 'Class' in nummerical values ######
df['Node_Status'].unique()
df['Class'].unique()
df['Node_Status']=df['Node_Status'].map({"b'B'":0,'b"\'P NB\'"':1,"b'NB'":2})
df['Class']=df['Class'].map({'b"\'No Block\'"':0,'b"\'NB-No Block\'"':1,"b'NB-Wait'":2,"b'Block'":3})
df.head(5)

############ plotting to check for Outlier using box plot #######
df.to_csv('Dataset_Rearranged.csv', index=False)


from spacy.en import English

parser = English()

def extract_entities(str):
    entityToBeSave = ""
    parsedSentence = parser(str)
    entities = list(parsedSentence.ents)
    for entity in entities:
        entityToBeSave += "{1} -> {0} | ".format(entity.label_, ' '.join([t.orth_ for t in entity]))
    return entityToBeSave


from Bio import SeqIO
def get_known_cluster_blast(bgc):
	if hasattr(bgc,'antismash_file'):
		bgc_file = bgc.antismash_file
		if not bgc_file:
			print("No antismash file in {}".format(bgc))
			return [] # Stops it being repeatedly called
		records = list(SeqIO.parse(bgc.antismash_file,'genbank'))
		hits = []
		for record in records:
			for feature in record.features:
				if feature.type == 'cluster':
					bgc_list = feature.qualifiers.get('knownclusterblast',None)
					if bgc_list:
						for bgc_item in bgc_list:
							tokens = bgc_item.split('\t')
							bgc_name = tokens[0].split('.')[1].strip()
							similarity = tokens[1].split('(')[1].split('%')[0]
							hits.append((bgc_name,similarity))
		return hits


import django_tables2 as tables
from models import Servers

class ServersTable(tables.Table):
    class Meta:
        model = Servers
        attrs = {'class': 'paleblue'}
        order_by = ('Program', 'Environment', 'ServerName')
        fields = ('ServerName', 'Environment', 'Program', 'Function',
                  'Location', 'Pysical_or_Virtual')


from SearchUtils import *

#####################################################
# SearchController.py by Matthew Terneus <mrtf38@mst.edu>
# Updated 2/8/12
# Contains classes: SearchController, StandardSearch,
# IterativeSearch, BiDirectionalSearch, IterativeBiDirectionalSearch
# Used to Control the search loop through the frontier of search tree
# Dependancies: SearchUtils
#####################################################

#### SearchController ABSTRACT class from SearchController.py
# Updated 2/4/12
# Contains methods search the frontier and return a usable result
class SearchController:
    ## run from SearchController
    # Links the search and parse
    # Args: Node start, list[Node] goal, Frontier searchtype, Anything args
    # Returns: tuple(list[tuple(action,state)],cost)
    def run(self,start,goal,searchtype,args = None):
        return self.parseBranch(self.search(start,goal,searchtype,args))

    ## search ABSTRACT from SearchController
    # main loop for searching the frontier
    # Args: Node start, list[Node] goal, Frontier searchtype, Anything args
    # Returns: Node containing goal
    def search(self,start,goal,searchtype,args = None):
        return None

    ## parseBranch from SearchController
    # Backtracks up tree to find the solution
    # Args: Node endnode
    # Returns: tuple(list[tuple(action,state)],cost)
    def parseBranch(self,endnode):
        if endnode == None:
            return (("Failure",""),0)
        crntnode = endnode
        actionset = []
        #while the root of the tree has not been passed
        while crntnode != None:
            #put action and state on the front of the list
            actionset.insert(0,(crntnode.action,crntnode.state.toStr()))
            crntnode = crntnode.parent
        return (actionset,endnode.cost)

#####################################################################################

#### StandardSearch class from SearchController.py
# Updated 2/4/12
# Contains methods search the frontier and return a usable result
# Inherits: SearchController
class StandardSearch(SearchController):
    ## search  from StandardSearch
    # main loop for searching the frontier
    # Args: Node start, list[Node] goal, Frontier searchtype, unused args
    # Returns: Node containing goal
    def search(self,start,goal,searchtype,args = None):
        #create root node
        root = Node(start,None,"Start",0)
        #initialize frontier as type searchtype
        frontier = searchtype(root,None,args)
        #Create Counter for nodes explored to compare effectiveness of huerisitics
        self.nodecount = 0
        #while nodes can be explored
        while(not frontier.empty()):
            # Pick Node
            node = frontier.pop()
            self.nodecount = self.nodecount+1
            # Check if goal
            for each in goal:
                if each.equals(node.state):
                    # Return end of branch for parsing
                    return node
            # expand new nodes into the frontier
            node.expand(frontier)
        #if no more nodes to explore search failed
        return None

#####################################################################################

#### IterativeSearch class from SearchController.py
# Updated 2/4/12
# Contains methods search the frontier iteratively and return a usable result
# Inherits: SearchController
class IterativeSearch(SearchController):
    ## search  from IterativeSearch
    # main loop for searching the frontier
    # Args: Node start, list[Node] goal, Frontier searchtype, depthlimit args
    # Returns: Node containing goal
    def search(self,start,goal,searchtype,args = None):
        #create root node
        root = Node(start,None,"Start",0)
        depth = 0
        #Create Counter for nodes explored to compare effectiveness of huerisitics
        self.nodecount = 0
        while True:
            #initialize frontier as type searchtype
            frontier = searchtype(root,depth,args)
            #while nodes can be explored
            while(not frontier.empty()):
                # Pick Node
                node = frontier.pop()
                self.nodecount = self.nodecount+1
                # Check if goal
                for each in goal:
                    if each.equals(node.state):
                        # Return end of branch for parsing
                        return node
                # expand new nodes into the frontier
                node.expand(frontier)
            #if no more nodes to explore search failed increase depth
            if hasattr(frontier,"depthstep"):
                # Use built in increment if availiable
                depth = frontier.depthstep
                if depth == -1:
                    break
            else:
                depth = depth + 1
        return None

#####################################################################################

#### BiDirectionalSearch class from SearchController.py
# Updated 2/4/12
# Contains methods search the frontier in two directions and return a usable result
# Inherits: SearchController
class BiDirectionalSearch(SearchController):
    ## search  from BiDirectionalSearch
    # main loop for searching the frontier
    # Args: Node start, list[Node] goal, Frontier searchtype, unused args
    # Returns: Node containing goal
    def search(self,start,goal,searchtype,args = None):
        # create root node
        root = Node(start,None,"Start",0)
        # initialize frontiers as type searchtype
        front = searchtype(root,None,args)
        back = searchtype(None,None,args)
        # fill back frontier with all goals
        for each in goal:
            newnode = Node(each,None,"End",0)
            back.push(newnode)

        # while nodes can be explored
        while (not front.empty()) and (not back.empty()):
            # Pick Node
            forward = front.pop()
            backward = back.pop()
            # check if the picked node is linked to the other frontier
            for each in back.toList():
                if each.state.equals(forward.state):
                    return self.splice(forward, each)
            for each in front.toList():
                if each.state.equals(backward.state):
                    return self.splice(each, backward)
            forward.expand(front)
            backward.expandBackward(back)
        #if no more nodes to explore search failed
        return None

    ## splice  from BiDirectionalSearch
    # combines a forward and backward brach into one forward branch
    # Args: Node start, list[Node] goal, Frontier searchtype, unused args
    # Returns: Node containing goal
    def splice(self,frontbranch, backbranch):
        nextnode = backbranch.parent
        lastnode = frontbranch
        lastaction = backbranch.action
        # while there is a parent on the backward branch
        while nextnode!= None:
            #shuffle parents and actions
            currentnode = nextnode
            nextaction = currentnode.action
            currentnode.action = lastaction
            lastaction = nextaction
            nextnode = currentnode.parent
            currentnode.parent = lastnode
            lastnode = currentnode
        #store the net path cost on the last node so that it can be returned with the solution
        lastnode.cost = frontbranch.cost + backbranch.cost
        return lastnode

#####################################################################################

#### IterativeBiDirectionalSearch class from SearchController.py
# Updated 2/4/12
# Contains methods search the frontier in two directions iteratively and return a usable result
# Inherits: BiDirectionalSearch
class IterativeBiDirectionalSearch(BiDirectionalSearch):
    ## search  from IterativeBiDirectionalSearch
    # main loop for searching the frontier
    # Args: Node start, list[Node] goal, Frontier searchtype, depthlimit args
    # Returns: Node containing goal
    def search(self,start,goal,searchtype,args = None):
        depth = 0
        while True:
            #create root node
            root = Node(start,None,"Start",0)
            # initialize frontiers as type searchtype
            front = searchtype(root,depth,args)
            back = searchtype(None,depth,args)
            # fill back frontier with all goals
            for each in goal:
                newnode = Node(each,None,"End",0)
                back.push(newnode)

            while (not front.empty()) and (not back.empty()):
                # Pick Nodes
                forward = front.pop()
                backward = back.pop()
                # check if the picked node is linked to the other frontier
                for each in back.toList():
                    if each.state.equals(forward.state):
                        return self.splice(forward, each)
                for each in front.toList():
                    if each.state.equals(backward.state):
                        return self.splice(each, backward)
                forward.expand(front)
                backward.expandBackward(back)
            #if no more nodes to explore search failed try deeper
            depth = depth+1
        return None