import discord
from discord_slash import SlashCommand
import os
from discord.ext import commands
import requests
from bs4 import BeautifulSoup

client = commands.Bot(command_prefix=[' '])
client.remove_command('help')
slash = SlashCommand(client, sync_commands=True) # Declares slash commands through the client.
token = os.environ.get('token')
guild_ids = [945418074121715782, 814607316107395093, 704482683857928222,961863482867847188, 962033686046466070]
roboteventskey = os.environ.get('robotevents_key')
def get_current_season():
	iq_seasons = 'https://www.robotevents.com/api/v2/seasons?program%5B%5D=41'
	headers = {"Authorization": f"Bearer {roboteventskey}"}
	seasondata = requests.get(iq_seasons, headers=headers).json()
	currentseasonid = seasondata['data'][1]['id']
	currentseasonname = seasondata['data'][1]['name']
	return currentseasonid, currentseasonname

currentseasonid, currentseasonname = get_current_season()

async def get_ranking(team):
	url = f'https://www.robotevents.com/teams/VIQC/{team.upper()}'
	res = requests.get(url)
	html_page = res.content
	soup = BeautifulSoup(html_page, 'html.parser')
	text = soup.find_all(text=True)

	output = ''
	blacklist = [
		'[document]',
		'noscript',
		'header',
		'html',
		'meta',
		'head',
		'input',
		'script',
	]

	for t in text:
		if t.parent.name not in blacklist:
			output += '{} '.format(t)
	index = output.find("World Skills Rank:")
	rank = ''
	for x in range(0, 4):
		if output[index+21+x].isnumeric():
			rank+=output[index+21+x]
	return rank
@client.event
async def on_ready():
	print("Ready!")
	# channel = client.get_guild(945418074121715782).get_channel(992529252652158977)
	# e = await channel.create_webhook(name='counting').url
	# print(e)
	game = discord.Game("a skills run")
	await client.change_presence(activity=game)
 # Put your server ID in this array.
guild_ids = [945418074121715782, 814607316107395093, 704482683857928222,961863482867847188, 962033686046466070]
# This command is used to get the ranking of a VIQC team.

@slash.slash(name='skills', description='World Skills Rank')
async def skills(ctx, team):

	team = team.upper()
	await ctx.defer()
	rank = await get_ranking(team)
	teaminfo = f'https://www.robotevents.com/api/v2/teams?number={team}'
	headers = {"Authorization": f"Bearer {roboteventskey}"}
	teamdata = requests.get(teaminfo, headers=headers).json()

	try:
		for allteam in teamdata['data']:
			if allteam['program']['code'] == 'VIQC':
				id = allteam['id']
				teamname = allteam['team_name']
		teaminfo = f'https://www.robotevents.com/api/v2/teams/{id}/skills'
	except:
		await ctx.send(f"Sorry, we couldn't find team {team}.")
		return

	driverlist = []
	autonlist = []
	events = {}

	def eventdata(teaminfo, driverlist, autonlist, events):
		teamdata = requests.get(teaminfo, headers=headers).json()
		for event in teamdata['data']:
			if event['season']['id'] == currentseasonid:
				if event['event']['name'] not in events:
					events[event['event']['name']] = {}
					events[event['event']['name']]['total'] = 0
					events[event['event']['name']]['driver'] = 0
					events[event['event']['name']]['auton'] = 0
				if event['type'] == 'driver':
					driverlist.append(event['score'])
					events[event['event']['name']]['total'] += event['score']
					events[event['event']['name']]['driver'] = event['score']
				elif event['type'] == 'programming':
					autonlist.append(event['score'])
					events[event['event']['name']]['total'] += event['score']
					events[event['event']['name']]['auton'] = event['score']
		if isinstance(teamdata['meta']['next_page_url'], str):
			teaminfo=teamdata['meta']['next_page_url']
			eventdata(teaminfo, driverlist, autonlist, events)
		return driverlist, autonlist, events

	eventdata(teaminfo, driverlist, autonlist, events)

	if driverlist:
		driverHigh = max(driverlist)
	else:
		driverHigh=0
	if autonlist:
		autonomousHigh = max(autonlist)
	else:
		autonomousHigh=0
	score = 0

	for total in events:
		if events[total]['total'] > score:
			score = events[total]['total']
			event = total
	try:
		autonomous = events[event]['auton']
	except:
		autonomous = 0
	try:
		driver = events[event]['driver']
	except:
		driver = 0

	embed=discord.Embed(title="Current VIQC Event", url="https://www.vexrobotics.com/iq/competition/viqc-current-game", color=0x24c9ff)
	embed.set_author(name=f"VIQC {team} ({teamname})", url=f"https://www.robotevents.com/teams/VIQC/{team}")
	embed.add_field(name="Rank", value=rank, inline=True)
	embed.add_field(name="Score", value=score, inline=True)
	embed.add_field(name="Programming", value=autonomous, inline=True)
	embed.add_field(name="Driver", value=driver, inline=True)
	embed.add_field(name="Highest Programming", value=autonomousHigh, inline=True)
	embed.add_field(name="Highest Driver", value=driverHigh, inline=True)
	print(embed)
	try:
		await ctx.send(embed=embed)
	except:
		await ctx.send(f"Hmm, we couldn't fetch your skills rank. Check https://www.robotevents.com/robot-competitions/vex-iq-challenge/standings/skills to see if {team} ({teamname}) is on there.")

@slash.slash(name='about', description='Bot credits')
async def credits(ctx):
	await ctx.send('''Made by VIQC 84C in 2022:
<@!697535361315766322>
<@!697913907528073296>
<@!475315771086602241>

Special thanks to makers of <@!329389228398215179>

If you would like to add the slash commands to your server, click on my profile and press "Add to Server"!''',
	allowed_mentions=discord.AllowedMentions(
	users=False,		 # Whether to ping individual user @mentions
	everyone=False,	  # Whether to ping @everyone or @here mentions
	roles=False,		 # Whether to ping role @mentions
	replied_user=False,  # Whether to ping on replies to messages
), hidden = True)

# This command is used to get the awards of a VIQC team.

@slash.slash(name='awards', description='Specified Team\'s awards')
async def awards(ctx, team):

	team = team.upper()
	await ctx.defer()
	teaminfo = f'https://www.robotevents.com/api/v2/teams?number={team}'
	headers = {"Authorization": f"Bearer {roboteventskey}"}
	teamdata = requests.get(teaminfo, headers=headers).json()

	try:
		for allteam in teamdata['data']:
			if allteam['program']['code'] == 'VIQC':
				id = allteam['id']
				teamname = allteam['team_name']
		awardsUrl = f'https://www.robotevents.com/api/v2/teams/{id}/awards?season={currentseasonid}'
	except:
		await ctx.send(f"Sorry, we couldn't find team {team}.")
		return

	headers = {"Authorization": f"Bearer {roboteventskey}"}

	# Award Data Loop for Next Page

	awardsDict = {}
	def awardData(awardsUrl, awardsDict):

		teamAwards = requests.get(awardsUrl, headers=headers).json()
		print(awardsUrl)

		for event in teamAwards['data']:
			if event['event']['name'] not in awardsDict:
				awardsDict[event['event']['name']] = []
			awardsDict[event['event']['name']].append(event['title'])
		if isinstance(teamAwards['meta']['next_page_url'], str):
			print('sus')
			awardsUrl=teamAwards['meta']['next_page_url']
			awardData(awardsUrl, awardsDict)
		return awardsDict

	awardData(awardsUrl, awardsDict)
	orderedAwards = ''

	for event in awardsDict:
		orderedAwards+=f'\n\n**{event}**\n'
		for award in awardsDict[event]:
			orderedAwards+=f"{award}\n"

	if not orderedAwards:
		orderedAwards = "\n\nCouldn't find any awards this season!"

	await ctx.send(f"**{team.upper()} ({teamname})'s Awards**{orderedAwards}")

@slash.slash(name='compare', description="Compare two teams by skills ranking")
async def compare(ctx, team1, team2):

	team1 = team1.upper()
	team2 = team2.upper()
	await ctx.defer()
	rank1 = await get_ranking(team1)
	rank2 = await get_ranking(team2)
	team1info = f'https://www.robotevents.com/api/v2/teams?number={team1}'
	team2info = f'https://www.robotevents.com/api/v2/teams?number={team2}'
	headers = {"Authorization": f"Bearer {roboteventskey}"}
	team1data = requests.get(team1info, headers=headers).json()
	team2data = requests.get(team2info, headers=headers).json()

	try:
		for allteam in team1data['data']:
			if allteam['program']['code'] == 'VIQC':
				id1 = allteam['id']
				team1name = allteam['team_name']
		for allteam2 in team2data['data']:
			if allteam['program']['code'] == 'VIQC':
				id2 = allteam2['id']
				team2name = allteam2['team_name']
	except:
		await ctx.send("Sorry, we couldn't find a team specified.")
		return

	team1info = f'https://www.robotevents.com/api/v2/teams/{id1}/skills'
	driverlist1 = []
	autonlist1 = []
	events1 = {}
	team2info = f'https://www.robotevents.com/api/v2/teams/{id2}/skills'
	driverlist2 = []
	autonlist2 = []
	events2 = {}

	def eventdata(teaminfo, driverlist, autonlist, events):
		teamdata = requests.get(teaminfo, headers=headers).json()
		for event in teamdata['data']:
			if event['season']['id'] == currentseasonid:
				if event['event']['name'] not in events:
					events[event['event']['name']] = {}
					events[event['event']['name']]['total'] = 0
					events[event['event']['name']]['driver'] = 0
					events[event['event']['name']]['auton'] = 0
				if event['type'] == 'driver':
					driverlist.append(event['score'])
					events[event['event']['name']]['total'] += event['score']
					events[event['event']['name']]['driver'] = event['score']
				elif event['type'] == 'programming':
					autonlist.append(event['score'])
					events[event['event']['name']]['total'] += event['score']
					events[event['event']['name']]['auton'] = event['score']
		if isinstance(teamdata['meta']['next_page_url'], str):
			teaminfo=teamdata['meta']['next_page_url']
			eventdata(teaminfo, driverlist, autonlist, events)
		return driverlist, autonlist, events

	eventdata(team1info, driverlist1, autonlist1, events1)
	eventdata(team2info, driverlist2, autonlist2, events2)

	if driverlist1:
		driverHigh1 = max(driverlist1)
	else:
		driverHigh1=0
	if autonlist1:
		autonomousHigh1 = max(autonlist1)
	else:
		autonomousHigh1=0
	score1 = 0

	for total in events1:
		if events1[total]['total'] > score1:
			score1 = events1[total]['total']
			event = total
	try:
		autonomous1 = events1[event]['auton']
	except:
		autonomous1 = 0
	try:
		driver1 = events1[event]['driver']
	except:
		driver1 = 0

	if driverlist2:
		driverHigh2 = max(driverlist2)
	else:
		driverHigh2=0
	if autonlist2:
		autonomousHigh2 = max(autonlist2)
	else:
		autonomousHigh2=0
	score2 = 0

	for total in events2:
		if events2[total]['total'] > score2:
			score2 = events2[total]['total']
			event = total
	try:
		autonomous2 = events2[event]['auton']
	except:
		autonomous2 = 0
	try:
		driver2 = events2[event]['driver']
	except:
		driver2 = 0

	rank1 = int(rank1)
	rank2 = int(rank2)
	if rank1 < rank2:
		rankdif = rank2-rank1
		r1 = f"{rankdif} ⬆🟩"
		r2 = f"{rankdif} ⬇🟥"
	else:
		rankdif = rank1-rank2
		r2 = f"{rankdif} ⬆🟩"
		r1 = f"{rankdif} ⬇🟥"
	if driver1 > driver2:
		driverdif = driver1-driver2
		d1 = f"{driverdif} ⬆🟩"
		d2 = f"{driverdif} ⬇🟥"
	elif driver2 > driver1:
		driverdif = driver2-driver1
		d2 = f"{driverdif} ⬆🟩"
		d1 = f"{driverdif} ⬇🟥"
	else:
		d1 = "↔⏺"
		d2 = "↔⏺"
	if autonomous1 > autonomous2:
		autonomousdif = autonomous1-autonomous2
		a1 = f"{autonomousdif} ⬆🟩"
		a2 = f"{autonomousdif} ⬇🟥"
	elif autonomous2 > autonomous1:
		autonomousdif = autonomous2-autonomous1
		a2 = f"{autonomousdif} ⬆🟩"
		a1 = f"{autonomousdif} ⬇🟥"
	else:
		a1 = "↔⏺"
		a2 = "↔⏺"

	if score1 > score2:
		scoredif = score1-score2
		s1 = f"{scoredif} ⬆🟩"
		s2 = f"{scoredif} ⬇🟥"
	elif score2 > score1:
		scoredif = score2-score1
		s2 = f"{scoredif} ⬆🟩"
		s1 = f"{scoredif} ⬇🟥"
	else:
		s1 = "↔⏺"
		s2 = "↔⏺"
	if autonomousHigh1 > autonomousHigh2:
		ahdif = autonomousHigh1-autonomousHigh2
		ah1 = f"{ahdif} ⬆🟩"
		ah2 = f"{ahdif} ⬇🟥"
	elif autonomousHigh2 > autonomousHigh1:
		ahdif = autonomousHigh2-autonomousHigh1
		ah2 = f"{ahdif} ⬆🟩"
		ah1 = f"{ahdif} ⬇🟥"
	else:
		ah1 = "↔⏺"
		ah2 = "↔⏺"
	if driverHigh1 > driverHigh2:
		addif = driverHigh1-driverHigh2
		ad1 = f"{addif} ⬆🟩"
		ad2 = f"{addif} ⬇🟥"
	elif driverHigh2 > driverHigh1:
		addif = driverHigh2-driverHigh1
		ad2 = f"{addif} ⬆🟩"
		ad1 = f"{addif} ⬇🟥"
	else:
		ad1 = "↔⏺"
		ad2 = "↔⏺"

	embed=discord.Embed(title="Current VIQC Event", url="https://www.vexrobotics.com/iq/competition/viqc-current-game", color=0x24c9ff)
	embed.set_author(name=f"Comparing VIQC {team1} vs. {team2}")
	embed.add_field(name=f"{team1} ({team1name}):", value=f"Rank: {rank1} ({r1})\nScore: {score1} ({s1})\nProgramming: {autonomous1} ({a1})\nDriver: {driver1} ({d1})\nHighest Programming: {autonomousHigh1} ({ah1})\nHighest Driver: {driverHigh1} ({ad1})")
	embed.add_field(name=f"{team2} ({team2name}):", value=f"Rank: {rank2} ({r2})\nScore: {score2} ({s2})\nProgramming: {autonomous2} ({a2})\nDriver: {driver2} ({d2})\nHighest Programming: {autonomousHigh2} ({ah2})\nHighest Driver: {driverHigh2} ({ad2})")
	await ctx.send(embed=embed)
# keep_alive.keep_alive()
client.run(token)


"""empty message

Revision ID: d22467bc4180
Revises: 60989a26e0c4
Create Date: 2021-12-13 10:40:14.209539

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd22467bc4180'
down_revision = '60989a26e0c4'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('task', sa.Column('is_bonus', sa.Boolean(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('task', 'is_bonus')
    # ### end Alembic commands ###


from flask import Blueprint
from flask import current_app as app

import os
import docker
import yaml
import paramiko

auto_update = Blueprint("auto_update", __name__)

@auto_update.route("/version/<opt>")
def version_opt(opt):
    client = docker.from_env()

    # 是否有更新
    FLAG = False

    # 获取yaml文件内容
    yaml_path = app.config.get("YAML_PATH")
    local_run_path = app.config.get("LOCAL_RUN_PWD")
    with open(yaml_path, encoding='utf-8') as f:
        yaml_dict = yaml.safe_load(f)

    # 将version号替换stable -> beta
    for _, svc in yaml_dict["services"].items():
        # 当前运行镜像的short_id
        print(svc["image"])
        # current_image_id = client.images.get(svc["image"]).short_id
        current_container = client.containers.get(svc["container_name"])
        print(current_container)

        # 更改待运行镜像版本
        version_replace = ["stable", "beta"] if opt == "up" else ["beta", "stable"]
        svc["image"] = svc["image"].replace(*version_replace)

        # 更新镜像
        print(f"Pull image {svc['image']}")
        new_image = client.images.pull(svc["image"])
        print(new_image.short_id)
        # 有需要更新的镜像
        if current_container.image.short_id != new_image.short_id and not FLAG:
            FLAG = True
    print(FLAG)
    if FLAG:
        # 将改变写入yaml文件
        with open(yaml_path, 'w', encoding='utf-8') as f:
            yaml.dump(yaml_dict, f, default_flow_style=False)

        # 重启docker-compose
        # ssh创建连接
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname=app.config.get("LOCAL_HOST"), port=22, username='root')
        print(f'cd {local_run_path} && docker-compose -f {"." + yaml_path} up -d')
        ssh.exec_command(
            f'cd {local_run_path} && docker-compose -f {"." + yaml_path} up -d')
        ssh.close()

    return "ok"


import requests
import json

from ..config.config import Config

class PokeAPI():
    def get_berry_data(self, berry_data_url):
        try:
            print(f"Fetching {berry_data_url}")
            raw_berry_data = json.loads(requests.get(berry_data_url).text)
            return {"name": raw_berry_data["name"], "growth_time": raw_berry_data["growth_time"]}
        except Exception as e:
            print(e)

    def get_berry_url_list(self, limit: int, offset: int):
        try:
            print(f"Fetching list with offset: {offset}")
            response = requests.get(f'{Config.POKE_API_URL}/berry/',
                                    params={"offset": offset, "limit": limit})
            json_response = json.loads(response.text)
            berry_url_list = json_response["results"]
            total_count = int(json_response["count"])
            return {"list": berry_url_list, "total_count": total_count}
        except Exception as e:
            print(e)


#0007_interprete_foto.py

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('appTop100', '0006_auto_20231206_1248'),
    ]

    operations = [
        migrations.AddField(
            model_name='interprete',
            name='foto',
            field=models.ImageField(blank=True, null=True, upload_to='fotos/'),
        ),
    ]


import logging
import os
import pickle
from typing import List

from client.gitgist import GitGist

HISTORY = os.path.join(os.path.dirname(__file__), 'history.pkl')
GISTS = os.path.join(os.path.dirname(__file__), 'gists.pkl')

def save_gists(update: List[GitGist]) -> None:
    if update:
        gists = load_gists()
        gists += update

        with open(GISTS, 'wb') as o:
            pickle.dump(gists, o, protocol=pickle.HIGHEST_PROTOCOL)
            logging.debug('Gists saved.')

def load_gists() -> List[GitGist]:
    try:
        with open(GISTS, 'rb') as i:
            return pickle.load(i)
    except (FileNotFoundError, EOFError):
        return []

def save_log(update: dict) -> None:
    if update:
        history = load_log()

        for user, last_checked_at in update.items():
            history[user] = last_checked_at

        with open(HISTORY, 'wb') as o:
            pickle.dump(history, o, protocol=pickle.HIGHEST_PROTOCOL)
            logging.debug("Request log updated.")

def load_log() -> dict:
    try:
        with open(HISTORY, 'rb') as i:
            history = pickle.load(i)

            for user, last_checked_at in history.items():
                history[user] = last_checked_at

            return history
    except (FileNotFoundError, EOFError):
        return {}


import json
import string
from time import sleep

import requests

SEARCH_URL = "https://api.chambers.com/api/search/name"
# ORG_PROFILE_URL = "https://api.chambers.com/api/organisations/23254203/profile-basics"
# ORG_PROVIDED_PROFILES_URL = "https://api.chambers.com/api/organisations/23254203/provided-profiles"
# ORG_DEPARTMENTS_URL = "https://api.chambers.com/api/organisations/23254203/ranked-departments"
# ORG_LAWYERS_URL = "https://api.chambers.com/api/organisations/23254203/ranked-lawyers"

SUPPORTED_GUIDES = ["USA"]

# map Description: Id
GUIDE_INFO = {
    "Global": {"id": 2, "description": "Global"},
    "USA": {"id": 5, "description": "USA"},
    "UK": {"id": 1, "description": "UK"},
    "Latin America": {"id": 9, "description": "Latin America"},
    "Asia-Pacific": {"id": 8, "description": "Asia-Pacific"},
    "Greater China Region": {"id": 116, "description": "Greater China Region"},
    "High Net Worth": {"id": 21, "description": "High Net Worth"},
    "FinTech": {"id": 49, "description": "FinTech"},
}

def index_firms():
    """index all of the firm ids (called organization ids) into a file 'firms.json'

    Only index firms from the currently supported guides,
    and save them based on which guide they belong to.

    Response Format:
    {'count': 19667, 'skip': 0, 'take': 5, 'results': [{'facetedSearchId': 11041, 'rowType': 'Firms', 'organisationName': 'Abell Eskew Landau', 'organisationId': 23254203, 'organisationLocation': 'New York', 'town': 'New York', 'country': 'USA', 'profilePictureUrl': 'https://media.chambers.com/organisations/392522/8b600834-87a5-45f3-8ec9-939c440776bb/profile.png', 'hasPaidProfile': True, 'publicationTypeGroups': [{'id': 5, 'description': 'USA'}]}]} # noqa: E501
    """
    PAGE_SIZE = 2000
    # DATA_FILENAME = "firms.json"
    # supported_guide_ids = [GUIDE_INFO[guide]["id"] for guide in SUPPORTED_GUIDES]
    body = {
        "query": "",
        "skip": 0,
        "take": PAGE_SIZE,
        "guideIds": [],
        "locationIds": [],
        "practiceAreaIds": [],
        "entityTypes": [],
    }

    for letter in string.ascii_lowercase[:4]:
        body["query"] = letter  # 'a'

        count = PAGE_SIZE + 9999  # make count larger than PAGE_SIZE
        skip = 0
        results = []

        # next page
        while skip < count:
            sleep(0.2)  # sleep 0.2 seconds
            response = requests.post(SEARCH_URL, json=body)
            data = response.json()
            results += data["results"]
            count = data["count"]
            skip += PAGE_SIZE
            body["skip"] = skip

        with open("data/aug15-2022/firms.json", "r") as firmsjson:
            firms = json.load(firmsjson)

        firms.extend(results)

        with open("data/aug15-2022/firms.json", "w") as firmsjson:
            json.dump(firms, firmsjson, indent=4)
            print(f"appended the letter {letter}")

        sleep(5)

index_firms()


from django.template import Library

register = Library()

ERRORS = ['error', 'reject', 'rejected']
SUCCESSES = ['accept', 'confirmed', 'fully-paid', 'shipped', 'refunded']

@register.inclusion_tag('status_label.html')
def render_status(status, status_display=None):
    if status in ERRORS:
        label_cls = 'danger'
    elif status in SUCCESSES:
        label_cls = 'success'
    else:
        label_cls = 'default'
    return {'label_cls': label_cls, 'status': status_display or status}


# See: https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service

import os

from flask import Flask

app = Flask(__name__)

@app.route("/")
def hello_world():
    name = os.environ.get("NAME", "World")
    return "Hello {}!".format(name)

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080)))


# /usr/bin/env python2
# -*- coding: utf-8 -*-

from __future__ import unicode_literals, print_function

import sys, os
from threading import Thread
import logging

logging.basicConfig(format='%(asctime)s,%(msecs)03d %(levelname)s [%(name)s] {%(threadName)s} %(message)s',
                            level=logging.INFO)
logger = logging.getLogger(__name__)
# logger.setLevel(logging.DEBUG)

from IoticAgent import ThingRunner
from IoticAgent.Core.compat import Event

DUREX_DASH_MAC = '68:37:e9:c5:9f:ff'
SHEBA_DASH_MAC = 'fc:a6:67:a1:58:bb'

LIGHTS = ['upstairs_lamp', 'bedroom_lamp']

button_evt = Event()

class IoticDash(ThingRunner):

    def __init__(self, config=None):
        super(IoticDash, self).__init__(config=config)
        self.__on = True

    def on_startup(self):
        # Create snd_ctrl Thing
        self.__thing = self.client.create_thing("dash ctrl")
        with self.__thing.get_meta() as meta:
            meta.set_label("Tim's House RPI Dash button handler")
            meta.set_description("RPI Wifi sniffing for Dash")
            meta.set_location(52.526087, 0.391160)  # Southery White Bell Pub
        self.__thing.set_public(False)
        #
        self.__light_ctrl = self.__thing.attach(('lights', 'ctrl'))

    def main(self):
        global button_evt
        while True:
            if button_evt.is_set():
                for light in LIGHTS:
                    self.__light_ctrl.ask({'lamp': light, 'cmd': 'on' if self.__on else 'off'})
                self.__on = False if self.__on else True
                button_evt.clear()
            if self.wait_for_shutdown(2):
                return

# dashbutton.py - https://gist.github.com/mr-pj/75297864abef5c8f2d5c134be2656023#file-dashbutton-py

from pydhcplib.dhcp_network import *

netopt = {'client_listen_port':"68", 'server_listen_port':"67", 'listen_address':"0.0.0.0"}

class Server(DhcpServer):
    def __init__(self, options, dashbuttons):
        DhcpServer.__init__(self, options["listen_address"],
                                options["client_listen_port"],
                                options["server_listen_port"])
        self.dashbuttons = dashbuttons

    def HandleDhcpRequest(self, packet):
        mac = self.hwaddr_to_str(packet.GetHardwareAddress())
        self.dashbuttons.press(mac)

    def hwaddr_to_str(self, hwaddr):
        result = []
        hexsym = ['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']
        for iterator in range(6) :
            result += [str(hexsym[hwaddr[iterator]/16]+hexsym[hwaddr[iterator]%16])]
        return ':'.join(result)

class DashButtons():
    def __init__(self):
        self.buttons = {}

    def register(self, mac, function):
        self.buttons[mac] = function

    def press(self, mac):
        if mac in self.buttons:
            self.buttons[mac]()
            return True
        return False

# dashbutton.py - end

def button_press():
    print("Got Dash button press!")
    button_evt.set()

def main():
    dashbuttons = DashButtons()
    dashbuttons.register(DUREX_DASH_MAC, button_press)
    dashbuttons.register(SHEBA_DASH_MAC, button_press)
    server = Server(netopt, dashbuttons)

    runner = IoticDash('../cfg/ioticdash.ini')
    runner.run(background=True)

    while True:
        try:
            server.GetNextDhcpPacket()
        except KeyboardInterrupt:
            break

    runner.stop()

if __name__ == '__main__':
    main()


## -*- encoding: latin-1 -*-

from numpy import *
from einsum2 import einsum2
from scipy import optimize
import scipy as sc
import pyfftw
import h5py

### Set U, U', J, J' as in parameters file (reduce module dependency)
def Uval_set(u0, JU_ratio):
    J        = JU_ratio*u0
    u0_prime = u0 - 2*J
    J_prime  = J
    return u0, JU_ratio, u0_prime, J, J_prime

##########################################################################
##### Main code: Calculate Greens function within FLEX approximation #####
##########################################################################

class gfunction_calc:
    """
    Self consistency loop for Greens function, using FLEX approximation
    """
    def __init__(self,p,h,b):
        ##### Tag for convergence check
        self.tag = 'calc'

        ##### Set parameters for U convergence
        u0_delta = 0.01 #change this parameter for U convergence if not working
        conv_it  = 2
        self.u0_pval  = p.u0
        u0_store = -1
        u0_it    = 0

        ##### Set static quantities (unit matrix, Hamiltonian in (iw, k)-space)
        self.set_static_quantities(p,h,b)

        ##### Set initial round (bare quantities)
        ## Initial guess for mu, sigma
        if p.round_it == 0:
            print("G convergence from ZERO.", file=open(p.Logstr,'a'))
            self.mu = h.mu
            self.sigma = zeros((len(b.fm), p.nk, p.nwan, p.nwan))
        else:
            print("G convergence from pre converged G.", file=open(p.Logstr,'a'))
            with h5py.File(p.loadpath,'r') as file:
                self.sigma = file['gfunction/sigma'][()]
                self.mu    = file['gfunction/mu'][()]

        ## Set G(iwn, k), G(tau, r), Chi_0(iwn, k)
        self.set_gkio(p,h,b,self.mu)
        gkio_old = self.gkio
        self.set_grit(p,b)
        self.set_ckit(p,b)

        ##### Self consistency loop
        if p.mode == 'FLEX':
            div_check_param = self.Max_eigval_ChiU(p,h)

            while div_check_param >= 1 or p.u0 != self.u0_pval or u0_it == 0:

                ##### Check for 1-max(chi@U) > 0, otherwise decrease U
                ### ONLY eigval_magnetic >= 1 check included, not charge!
                # Safety check for too long running calculations
                u0_store = p.u0
                u0_it += 1
                if u0_it == 100:
                    print("U iteration reached step 100. Everything okay?", file=open(p.Logstr,'a'))
                    print(p.err_str_begin + "U iteration reached step 100", file=open(p.Logerrstr,'a'))
                if u0_it == 150:
                    print("U iteration reached step 150. It will be stopped!", file=open(p.Logstr,'a'))
                    print(p.err_str_begin + "U iteration reached step 150", file=open(p.Logerrstr,'a'))
                    break

                # If it's not good already after one cycle, reset U
                if p.u0 != self.u0_pval or u0_it > 1:
                    p.u0 = min(self.u0_pval, p.u0*1.5)
                    _, _, p.u0_prime, p.J, p.J_prime = Uval_set(p.u0, p.JU_ratio)
                    h.set_interaction(p)
                    div_check_param = self.Max_eigval_ChiU(p,h)
                    conv_it -= 10
                    conv_it  = max(conv_it, 1)

                ##### Setting new U if max(|chi0*U|) >= 1
                print('### Check for renormalization |chi@U_S|: ' + str(div_check_param) + ', U = ' + str(p.u0), file=open(p.Logstr,'a'))

                ckio_max = amax(linalg.eigh(self.ckio.reshape(len(b.bm),p.nk1,p.nk2,p.nk3,p.nwan**2,p.nwan**2))[0])
                while div_check_param >= 1:
                    while div_check_param/(div_check_param + u0_delta*conv_it)*1/ckio_max - self.u0_pval > 0.01:
                        conv_it += 5
                        #print(conv_it, div_check_param/(div_check_param + u0_delta*conv_it)*1/ckio_max)
                    p.u0 = min(self.u0_pval,\
                        div_check_param/(div_check_param + u0_delta*conv_it)*1/ckio_max)
                    #print(ckio_max, div_check_param/(div_check_param + u0_delta*conv_it)*1/ckio_max)

                    print("New U set to " + str(p.u0), conv_it)
                    if p.u0 == self.u0_pval:
                        conv_it += 1
                    else:
                        _, _, p.u0_prime, p.J, p.J_prime = Uval_set(p.u0, p.JU_ratio)
                        h.set_interaction(p)
                        div_check_param = self.Max_eigval_ChiU(p,h)
                        if div_check_param >=1:
                            conv_it += 1

                print('New U value: ' + str(p.u0) + ', with |chi@U_S|: ' + str(div_check_param), file=open(p.Logstr,'a'))

                # Safety check if U is the same value twice -> some error has occured!
                if p.u0 == u0_store and u0_it != 1:
                    print("Same U value as before!", file=open(p.Logstr,'a'))
                    print(p.err_str_begin + "Same U value reached twice, abbortion! No convergence! (U = "+str(p.u0) +")", file=open(p.Logerrstr,'a'))
                    break

                #--------------------------------------------------------------
                #### Convergence cycle of FLEX self-energy for given U
                # Setting of convergence tolerance and iteration number (U ~= U_in -> oneshot calculation)
                if p.u0 == self.u0_pval:
                    conv_tol   = p.g_sfc_tol
                    sfc_it_max = 150
                    mix        = p.mix
                else:
                    conv_tol   = 8e-2
                    sfc_it_max = 1
                    mix        = p.mix

                # Convergence loop
                print('Convergence round for U = ' + str(p.u0) + \
                      ',|chi@U_S|: ' + str(div_check_param),\
                      file=open(p.Logstr,'a'))
                for it_sfc in range(sfc_it_max):
                    self.sigma_old = self.sigma
                    self.set_V(p,h,b)
                    self.set_sigma(p,b)

                    self.set_mu_from_gkio(p,h,b)
                    self.set_gkio(p,h,b,self.mu)
                    self.symmetrize_gkio(p,b)

                    #Mixing: Change values if needed!
                    if p.u0 != 1:
                        self.gkio = mix*self.gkio + (1-mix)*gkio_old

                    gkio_old = self.gkio
                    self.set_grit(p,b)
                    self.set_ckit(p,b)

                    print(it_sfc, sum(abs(self.sigma_old-self.sigma))/sum(abs(self.sigma)),\
                          file=open(p.Logstr,'a'))
                    if sum(abs(self.sigma_old-self.sigma))/sum(abs(self.sigma)) <= conv_tol:
                        break

        ##### Security convergence check
        ### U convergence
        if p.u0 != self.u0_pval and p.mode == 'FLEX':
            print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\nU is not initial input. Stopping gfunction."\
                  , file=open(p.Logstr,'a'))
            print(p.err_str_begin + "U != U_init | gfunction stopped."\
                  , file=open(p.Logerrstr,'a'))
            return

        ### Sigma convergence
        if p.mode == 'FLEX' and sum(abs(self.sigma_old-self.sigma))/sum(abs(self.sigma)) > conv_tol:
            print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\nSigma not converged. Stopping gfunction."\
                  , file=open(p.Logstr,'a'))
            print(p.err_str_begin + "Sigma not converged (diff = "\
                  + str(sum(abs(self.sigma_old-self.sigma))/sum(abs(self.sigma)))\
                  + ") | gfunction stopped.", file=open(p.Logerrstr,'a'))
            return

        ### Calculate G function of negative k
        gkio_invk  = self.gkio.reshape(len(b.fm),p.nk1,p.nk2,p.nk3,p.nwan**2)
        fft_object = pyfftw.builders.fftn(gkio_invk,axes=(1,2,3))
        gkio_invk  = fft_object()
        fft_object = pyfftw.builders.fftn(gkio_invk,axes=(1,2,3))
        gkio_invk  = fft_object()/p.nk
        self.gkio_invk = gkio_invk.reshape(len(b.fm),p.nk,p.nwan,p.nwan)
        print("Self consistency loop finished!", file=open(p.Logstr,'a'))

        ### Calculate maximal magnetic/charge eigenvalue
        print("Extract largest BSE kernel values.", file=open(p.Logstr,'a'))
        BSE_max_spin   = self.Max_eigval_ChiU(p,h,'S')
        BSE_max_charge = self.Max_eigval_ChiU(p,h,'C')

        with open(p.BSE_EV_path,"a") as file:
            file.write('{} {} {}\n'.format(p.T, BSE_max_spin, BSE_max_charge))

        print("### Maximal BSE kernel value Chi@U_S = " + str(BSE_max_spin)\
              + " | Chi@U_C = " + str(BSE_max_charge), file=open(p.Logstr,'a'))

        ##### Finished loop, save results
        print("Saving all data now...", file=open(p.Logstr,'a'))

        with h5py.File(p.savepath,'a') as file:
            group = file.require_group('gfunction')

            group.require_dataset('gkio' , data=self.gkio , shape=self.gkio.shape, dtype=complex)
            group.require_dataset('sigma', data=self.sigma, shape=self.sigma.shape, dtype=complex)
            group.require_dataset('mu', data=self.mu, shape=(), dtype=float)

            group.require_dataset('BSE_max_spin'  , data = BSE_max_spin  , shape=(), dtype=float)
            group.require_dataset('BSE_max_charge', data = BSE_max_charge, shape=(), dtype=float)

        print("Done! -> Move to SC calculation.", file=open(p.Logstr,'a'))

    ##############
    # Calculate chemical potential via bisection method from G(k, iw_n).
    ##############

    ### Set from Greens function ----------------------------------------------
    def set_mu_from_gkio(self,p,h,basis):
        pass
        """
        Determining iteration mu from bisection method of 1 + sum(gkio)
        """

        ### Set electron number for bisection difference
        # n_0 is per electron orbital!
        n_0 = p.n_fill
        n = self.calc_electron_density_from_gkio
        f = lambda mu : 2*n(p,h,basis,mu) - n_0

        self.mu = sc.optimize.bisect(f, 50, -50)

    #--------------------------------------------------------------------------
    def calc_electron_density_from_gkio(self,p,h,b,mu):
        self.set_gkio(p,h,b,mu)
        gio = sum(self.gkio,axis=1)/p.nk
        gio = trace(gio,0,1,2)/p.norb #offset=0, axis=1,2

        n = 1 + real(dot(b.fermi_iw_to_tau_0, gio))
        return n

    ##############
    ### Set functions for self consistency loop.
    # set_gkio  : p, b | calculates G(k, iw_n)
    # set_grit  : p, b | calculates G(r, tau) via FFT + irbasis on bosonic tau
    # set_ckit  : p, b | calculates chi_0(k, iv_m) via G(r, tau) and FFT + irbasis
    # set_v     : p, b | calculates V(r, tau) on fermionic tau via chi_0, FFT + irbasis
    # set_sigma : p, b | calculates Sigma(k, iw_n) via V and G
    ##############

    ### Set static quantities
    def set_static_quantities(self, p, h, b):
        self.E_  = tensordot(ones(len(b.fm)), array([eye(h.hk.shape[1],h.hk.shape[2]) for it in range(h.hk.shape[0])]), axes=0)
        self.hk_ = tensordot(ones(len(b.fm)), h.hk, axes=0)
        self.io_ = b.fm.reshape(len(b.fm),1,1,1)*self.E_

        #For calculating V
        self.E_int = tensordot(ones(len(b.bm)), array([eye(p.nwan**2,p.nwan**2) for it in range(p.nk)]), axes=0)

    ### Set G(k, iw_n) --------------------------------------------------------
    def set_gkio(self, p, h, b, mu):
        self.gkio = linalg.inv(self.io_ + mu*self.E_ - self.hk_ - self.sigma)

    def symmetrize_gkio(self,p,b):
        # G(k, iwn) = G^T(-k, iwn)
        gkio_invk = roll(self.gkio.reshape(len(b.fm),p.nk1,p.nk2,p.nwan,p.nwan),-1,axis=(1,2))[:,::-1,::-1]
        gkio_invk = gkio_invk.reshape(len(b.fm),p.nk,p.nwan,p.nwan)
        self.gkio = (self.gkio + transpose(gkio_invk,axes=(0,1,3,2)))/2

    ### Set G(r, tau) ---------------------------------------------------------
    def set_grit(self, p, b):
        if not hasattr(self, 'gkio'): exit("Error_set_grit")
        grit = self.gkio.reshape(len(b.fm), p.nk1, p.nk2, p.nk3, p.nwan**2)

        fft_object = pyfftw.builders.fftn(grit, axes=(1,2,3))
        grit = fft_object()

        grit_0 = grit[:,0,0,0].reshape(len(b.fm),p.nwan**2)
        grit = grit.reshape(len(b.fm),p.nk*p.nwan**2)
        self.grit_b = dot(b.fermi_iw_to_tau_boson, grit).reshape(len(b.bt),p.nk,p.nwan,p.nwan)
        self.grit_f = dot(b.fermi_iw_to_tau,       grit).reshape(len(b.ft),p.nk,p.nwan,p.nwan)
        self.grit_f_0 = - dot(b.fermi_iw_to_tau_0, grit_0).reshape(p.nwan,p.nwan)

    ### Set chi_0(k, iv_m) ----------------------------------------------------
    def set_ckit(self, p, b):
        if not hasattr(self, 'grit_b'): exit("Error_set_ckit")
        grit_rev = self.grit_b[::-1,:,:,:]    #G_lm(r,beta-tau)
        ckio = einsum2('ijkm,ijln->ijklmn', self.grit_b, grit_rev).reshape(len(b.bt),p.nk*p.nwan**4)#km ln

        ckio = dot(b.bose_tau_to_iw, ckio)
        ckio = ckio.reshape(len(b.bm),p.nk1,p.nk2,p.nk3,p.nwan**4)

        fft_object = pyfftw.builders.ifftn(ckio, axes=(1,2,3))
        ckio = fft_object()/p.nk

        self.ckio = ckio.reshape(len(b.bm),p.nk,p.nwan**2,p.nwan**2)

    ### V(r, tau) -------------------------------------------------------------
    def set_V(self,p,h,b):
        if p.nspin == 1 and p.nwan >= 2:
            chi_spin   = linalg.inv(self.E_int - self.ckio@h.S_mat)@self.ckio
            chi_charge = linalg.inv(self.E_int + self.ckio@h.C_mat)@self.ckio

            V = 3./2.* h.S_mat@(chi_spin   - 1/2*self.ckio)@h.S_mat \
              + 1./2.* h.C_mat@(chi_charge - 1/2*self.ckio)@h.C_mat
        else:
            print('No multiorbital system detected! Use different script...'\
                  , file=open(p.Logstr,'a'))

        V = V.reshape(len(b.bm),p.nk1,p.nk2,p.nk3,p.nwan**4)
        fft_object = pyfftw.builders.fftn(V, axes=(1,2,3))
        V = fft_object().reshape(len(b.bm),p.nk*p.nwan**4)

        V      = dot(b.bose_iw_to_tau_fermi, V)
        self.V = V.reshape(len(b.ft),p.nk,p.nwan,p.nwan,p.nwan,p.nwan)

        V_DC = 3./2.*h.S_mat - 1./2.*h.C_mat
        self.V_DC = V_DC.reshape(p.nwan,p.nwan,p.nwan,p.nwan)

    ### Sigma(k, iw_n) --------------------------------------------------------
    def set_sigma(self,p,b):
        sigma = einsum('ijklmn,ijln->ijkm',self.V, self.grit_f)
        sigma = sigma.reshape(len(b.ft),p.nk*p.nwan**2)

        sigma = dot(b.fermi_tau_to_iw, sigma)
        sigma = sigma.reshape(len(b.fm),p.nk1,p.nk2,p.nk3,p.nwan**2)

        fft_object = pyfftw.builders.ifftn(sigma, axes=(1,2,3))
        sigma = fft_object()/p.nk
        self.sigma = sigma.reshape(len(b.fm),p.nk,p.nwan,p.nwan)

        sigma_DC   = einsum2('klmn,ln->km',self.V_DC , self.grit_f_0)*ones((len(b.fm),p.nk,p.nwan,p.nwan))/p.nk
        self.sigma = self.sigma + sigma_DC

    ##############
    # Function for calculating max eig(chi@U_S) for diverging check
    ##############

    def Max_eigval_ChiU(self, p, h, channel='S'):
        '''
        Calculate max{eig(ckio@S_mat)} as a measure for divergence check in chi_spin
        '''
        chan_dic = {'S': h.S_mat, 'C': h.C_mat, 'S_ask' : amax, 'C_ask': amin}

        X = self.ckio@chan_dic[channel]
        X = X.reshape(-1, p.nk1, p.nk2, p.nk3, p.nwan**2, p.nwan**2)
        #X_eig, _ = linalg.eigh(X)
        X_eig = linalg.eigvals(X)
        if abs(imag(chan_dic[channel+'_ask'](X_eig))) > 10**(-10):
                    print("!!!!!!! Imaginary part of chi eig_val is very large !!!!!!",file=open(p.Logstr,'a'))
        X_eig = real(X_eig)
        return chan_dic[channel+'_ask'](X_eig)

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------

##############
# Load Greens function if already calculated
##############

class gfunction_load:
    """
    Load already calculated and saved quantities
    """
    def __init__(self,p,b):
        ##### Tag for convergence check
        self.tag = 'load'

        ##### Full G already calculated, load results
        print("Self consistency loop already done. Loading data now...", file=open(p.Logstr,'a'))

        # G(iwn_fermi, k)
        with h5py.File(p.loadpath,'r') as file:
            self.gkio = file['gfunction/gkio'][()]
        print('G(io,k) done', end =" ", file=open(p.Logstr,'a'))

        # G(iwn_fermi, -k)
        gkio_invk  = self.gkio.reshape(len(b.fm),p.nk1,p.nk2,p.nk3,p.nwan**2)
        fft_object = pyfftw.builders.fftn(gkio_invk,axes=(1,2,3))
        gkio_invk  = fft_object()
        fft_object = pyfftw.builders.fftn(gkio_invk,axes=(1,2,3))
        gkio_invk  = fft_object()/p.nk
        self.gkio_invk = gkio_invk.reshape(len(b.fm),p.nk,p.nwan,p.nwan)
        print('| G(io,-k) [calc] done', end =" ", file=open(p.Logstr,'a'))

        # G(tau, r)
        grit = self.gkio.reshape(len(b.fm), p.nk1, p.nk2, p.nk3, p.nwan**2)
        fft_object = pyfftw.builders.fftn(grit, axes=(1,2,3))
        grit = fft_object()
        grit = grit.reshape(len(b.fm),p.nk*p.nwan**2)
        self.grit_b = dot(b.fermi_iw_to_tau_boson, grit).reshape(len(b.bt),p.nk,p.nwan,p.nwan)
        print('| G(tau,r) [calc] done', end =" ", file=open(p.Logstr,'a'))

        # chi_0(iwn_bose, k)
        grit_rev = self.grit_b[::-1,:,:,:]    #G_lm(r,beta-tau)
        ckio = einsum2('ijkm,ijln->ijklmn', self.grit_b, grit_rev).reshape(len(b.bt),p.nk*p.nwan**4)#km ln
        ckio = dot(b.bose_tau_to_iw, ckio)
        ckio = ckio.reshape(len(b.bm),p.nk1,p.nk2,p.nk3,p.nwan**4)
        fft_object = pyfftw.builders.ifftn(ckio, axes=(1,2,3))
        ckio = fft_object()/p.nk
        self.ckio = ckio.reshape(len(b.bm),p.nk,p.nwan**2,p.nwan**2)
        print('| Chi_0(iw,k) [calc] done', file=open(p.Logstr,'a'))

        ##### Interaction identity
        self.E_int = tensordot(ones(self.ckio.shape[0]), array([eye(p.nwan**2,p.nwan**2) for it in range(p.nk)]), axes=0)


from io import BytesIO

from pyrogram.raw.core.primitives import Int, Long, Int128, Int256, Bool, Bytes, String, Double, Vector
from pyrogram.raw.core import TLObject
from pyrogram import raw
from typing import List, Optional, Any

# # # # # # # # # # # # # # # # # # # # # # # #
#               !!! WARNING !!!               #
#          This is a generated file!          #
# All changes made in this file will be lost! #
# # # # # # # # # # # # # # # # # # # # # # # #

class ChatInviteImporter(TLObject):  # type: ignore
    """Telegram API type.

    Constructor of :obj:`~pyrogram.raw.base.ChatInviteImporter`.

    Details:
        - Layer: ``151``
        - ID: ``8C5ADFD9``

    Parameters:
        user_id (``int`` ``64-bit``):
            N/A

        date (``int`` ``32-bit``):
            N/A

        requested (``bool``, *optional*):
            N/A

        about (``str``, *optional*):
            N/A

        approved_by (``int`` ``64-bit``, *optional*):
            N/A

    """

    __slots__: List[str] = ["user_id", "date", "requested", "about", "approved_by"]

    ID = 0x8c5adfd9
    QUALNAME = "types.ChatInviteImporter"

    def __init__(self, *, user_id: int, date: int, requested: Optional[bool] = None, about: Optional[str] = None, approved_by: Optional[int] = None) -> None:
        self.user_id = user_id  # long
        self.date = date  # int
        self.requested = requested  # flags.0?true
        self.about = about  # flags.2?string
        self.approved_by = approved_by  # flags.1?long

    @staticmethod
    def read(b: BytesIO, *args: Any) -> "ChatInviteImporter":

        flags = Int.read(b)

        requested = True if flags & (1 << 0) else False
        user_id = Long.read(b)

        date = Int.read(b)

        about = String.read(b) if flags & (1 << 2) else None
        approved_by = Long.read(b) if flags & (1 << 1) else None
        return ChatInviteImporter(user_id=user_id, date=date, requested=requested, about=about, approved_by=approved_by)

    def write(self, *args) -> bytes:
        b = BytesIO()
        b.write(Int(self.ID, False))

        flags = 0
        flags |= (1 << 0) if self.requested else 0
        flags |= (1 << 2) if self.about is not None else 0
        flags |= (1 << 1) if self.approved_by is not None else 0
        b.write(Int(flags))

        b.write(Long(self.user_id))

        b.write(Int(self.date))

        if self.about is not None:
            b.write(String(self.about))

        if self.approved_by is not None:
            b.write(Long(self.approved_by))

        return b.getvalue()


"""HASC dataset

http://hasc.jp/
"""

import re
import warnings
import functools
from pathlib import Path
from typing import List, Tuple, Union, Optional, Iterable

import numpy as np
import pandas as pd
import pickle
from ..core import split_using_sliding_window

from .base import BaseDataset, check_path

__all__ = ['HASC', 'load', 'load_raw', 'load_meta']

class HASC(BaseDataset):
    """
    HASCデータセット(HASC-PAC2016)に記録されているセンサデータとメタデータを読み込む．

    Parameters
    ----------
    path: Path
        HASCデータセットのパス．BasicActivityディレクトリの親ディレクトリのパスを指定する．

    meta_cache_path: Optional[Path]
        メタデータのキャッシュファイルのパス．
        何も指定されない場合(meta_cache_path=None)，メタデータの作成を行うが，キャッシュファイルは作成しない．
        ファイル名が指定された場合，そのファイルが存在すればそこからメタデータを読み込み，存在しなければメタデータの作成を行い指定したファイルパスにダンプする．

    See Also
    --------
    メタデータの読み込みは非常に時間がかかるため，キャッシュファイルを活用することをおすすめする．
    """

    supported_queries = [
        'Frequency', 'Gender', 'Height', 'Weight', 'Person'
    ]

    supported_activity_labels = [
        'stay', 'walk', 'jog', 'skip', 'stup', 'stdown',
    ]

    supported_target_labels = ['activity', 'frequency', 'gender', 'height', 'weight', 'person']

    MAPS = {
        'activity': {'1_stay': 0, '2_walk': 1, '3_jog': 2, '4_skip': 3, '5_stUp': 4, '6_stDown': 5},
        'gender': {'male': 0, 'female': 1},
    }

    def __init__(self, path:Path, meta_cache_path:Optional[Path]=None):
        super().__init__(path)
        self.cache_dir_meta = meta_cache_path

        if meta_cache_path is not None:
            if meta_cache_path.exists():
                self.meta = pd.read_csv(str(meta_cache_path), index_col=0)
            else:
                self.meta = load_meta(path)
                self.meta.to_csv(str(meta_cache_path))
        else:
            self.meta = load_meta(path)

        dtypes = {
            'LogVersion': np.float64,
            'Frequency': np.float64,
            'Height(cm)': np.float64,
            'Weight(kg)': np.float64,
            'Pace(cm)': np.float64,
            'HeightOfOneStairStep(cm)': np.float64,
            'UseHistory': np.float64,
            'Count': np.float64,
            # 'LogVersion': np.float64,
        }
        self.meta = self.meta.replace('', np.nan)
        self.meta = self.meta.astype(dtypes)

        self.label_map = {'activity': None, 'subject': None}

    @classmethod
    def load_from_cache(cls, cache_path:Path):
        with cache_path.open('rb') as fp:
            x, y = pickle.load(fp)
        return x, y

    def _filter_with_meta(self, queries):
        """
        Supported Queries
        - Frequency
        - Gender
        - Weight
        - Person
        """
        if not (set(queries.keys()) < set(self.supported_queries)):
            raise ValueError(f'Unknown queries detected. (Supported: {self.supported_queries})')

        # クエリの整形
        # 括弧は特殊文字として扱われるため無視したい場合はバッククォートで囲む
        if 'Height' in queries.keys():
            queries['Height'] = queries['Height'].replace('Height', '`Height(cm)`')
        if 'Weight' in queries.keys():
            queries['Weight'] = queries['Weight'].replace('Weight', '`Weight(kg)`')
        if 'Person' in queries.keys():
            queries['Person'] = queries['Person'].replace('Person', 'person')
        for k in queries.keys():
            queries[k] = f'({queries[k]})'

        query_string = ' & '.join([queries[k] for k in queries.keys()])
        filed_meta = self.meta.query(query_string)
        return filed_meta

    def __extract_targets(self, y_labels:Iterable, meta_row:pd.Series) -> np.ndarray:
        if not isinstance(meta_row, pd.Series):
            raise TypeError('meta_row must be "pd.Series", but {}'.format(type(meta_row)))
        targets = []
        for i, yl in enumerate(y_labels):
            # metaデータのカラム名に変換
            ylbl2col = {
                'activity': 'act',
                'frequency': 'Frequency',
                'gender': 'Gender',
                'height': 'Height(cm)',
                'weight': 'Weight(kg)',
                'person': 'person',
            }
            assert set(ylbl2col.keys()) == set(self.supported_target_labels), 'table for converting "y_label" to column is not enough'

            # t = meta_row._asdict()[yl]
            t = meta_row.to_dict()[ylbl2col[yl]]
            cat_labels = ('activity', 'gender', 'person')
            if yl in cat_labels:
                if yl == 'person':
                    reg = re.compile(r'^person(?P<pid>\d+)$')
                    m = reg.fullmatch(t)
                    if m is not None:
                        pid = int(m.group('pid'))
                        self.maps[yl][t] = pid
                        targets += [pid]
                    else:
                        warnings.warn('Detected unknown label: "{}" of person label. Therefore, this data is ignored.'.format(t), Warning)
                        raise RuntimeError('invalid label')
                else:
                    if t not in HASC.MAPS[yl]:
                        warnings.warn('Detected unknown label: "{}" of {} label. Therefore, this data is ignored.'.format(t, yl), Warning)
                        raise RuntimeError('invalid label')
                    targets += [HASC.MAPS[yl][t]]

            elif yl in (set(self.supported_target_labels) - set(cat_labels)):
                targets += [t]
            # else:
            #     # not reach

        return np.array(targets)

    # フィルタリング周りの実装は暫定的
    def load(self, window_size:int, stride:int, ftrim:int=0, btrim:int=0, queries:Optional[dict]=None, y_labels:Union[str, list]='activity') -> Tuple[np.ndarray, np.ndarray]:
        """
        HASCデータセットを読み込み，sliding-window処理を行ったデータを返す．
        ここでは3軸加速度センサデータのみを扱う．

        Parameters
        ----------
        window_size: int
            フレーム分けするサンプルサイズ

        stride: int
            ウィンドウの移動幅

        ftrim: int
            セグメント先頭のトリミングサイズ

            ここで指定する値はトリミングする時系列センサデータのデータ長である．
            他のデータセットでは秒数を指定するが，HASCデータセットは異なるサンプリングレートのデータが混在しているため，秒数ではなくデータ長で指定する．
            btrimにおいても同様．

        btrim: int
            セグメント末尾のトリミングサイズ

        queries: dict
            メタ情報に基づいてフィルタリングを行うためのクエリ．

            Keyはフィルタリングのラベル(Supported: Frequency, Height, Weight, Gender)

            Valueはクエリ文字列(DataFrame.queryに準拠)

            詳しい使い方は後述．

        y_labels: Union[str, list]

            ターゲットデータとしてロードするデータの種類を指定．
            listで指定した場合，その順序が返されるターゲットラベルにも反映される．
            サポートする種類は以下の通り(今後拡張予定)．

            - 'activity'
            - 'frequency'
            - 'gender'
            - 'height'
            - 'weight'
            - 'person'

        Returns
        -------
        (x_frames, y_frames): Tuple[np.ndarray, np.ndarray]
            sliding-windowで切り出した入力とターゲットのフレームリスト．

            x_framesは3次元配列で構造は大まかに(Batch, Channels, Frame)のようになっている．
            Channelsは加速度センサの軸を表しており，先頭からx, y, zである．

            y_framesはy_labelsで指定したターゲットラベルであり，
            y_frames(axis=1)のラベルの順序はy_labelsのものが保持されている．

        Examples
        --------
        サンプリングレートが100Hz and 身長が170cmより大きい and 体重が100kg以上でフィルタリング
        >>> hasc_path = Path('/path/to/dataset/HASC-PAC2016/')  # HASCデータセットパス
        >>> hasc = HASC(hasc_path, Path('D:/datasets/HASC-PAC2016/BasicActivity/hasc.csv'))
        >>> queries = {
        >>>     'Frequency': 'Frequency == 100', # サンプリングレートが100Hzのデータのみを取得
        >>>     'Height': 'Height > 170',   # 身長が170cmより大きい人
        >>>     'Weight': 'Weight >= 100',   # 体重が100kg以上の人
        >>> }
        >>>
        >>> y_labels = ['activity', 'person']    # ターゲットラベルとしてacitivityとpersonを取り出す
        >>>
        >>> # yのaxis=1にはactivity, personの順でターゲットラベルが格納されている．
        >>> x, y, act2id = hasc.load(window_size=256, stride=256, queries=queries, ftrim=2*100, btrim=2*100, y_labels=y_labels)
        >>>
        >>> print(f'x: {x.shape}, y: {y.shape}')
        >>> # > x: (?, 3, 256), y: (?, 2)
        """

        if isinstance(y_labels, str):
            y_labels = [y_labels]
        if not (set(y_labels) <= set(self.supported_target_labels)):
            raise ValueError('include not supported target labels: {}'.format(y_labels))

        if queries is None:
            filed_meta = self.meta
        else:
            filed_meta = self._filter_with_meta(queries)

        segments, _ = load(self.path, meta=filed_meta)
        x_frames = []
        y_frames = []
        self.maps = dict(zip(y_labels, [{} for _ in y_labels]))
        for (_, meta_row), seg in zip(filed_meta.iterrows(), segments):
            act = meta_row['act']
            if act == '0_sequence':
                continue
            try:
                ys = self.__extract_targets(tuple(y_labels), meta_row)
            except RuntimeError as e:
                if str(e) == 'invalid label':
                    continue
                else:
                    raise e

            fs = split_using_sliding_window(
                np.array(seg), window_size=window_size, stride=stride,
                ftrim=ftrim, btrim=btrim,
                return_error_value=None)
            if fs is not None:
                x_frames += [fs]
                y_frames += [np.expand_dims(ys, axis=0).repeat(len(fs), axis=0)]
        x_frames = np.concatenate(x_frames)
        x_frames = x_frames.transpose(0, 2, 1)
        y_frames = np.concatenate(y_frames)
        assert len(x_frames) == len(y_frames), 'Mismatch length of x_frames and y_frames'

        self.label_map = self.maps.copy()
        self.label_map.update(HASC.MAPS.copy())

        return x_frames, y_frames, self.label_map

def load(path:Union[Path,str], meta:pd.DataFrame) -> Tuple[List[pd.DataFrame], pd.DataFrame]:
    """Function for loading HASC dataset

    Parameters
    ----------
    path: Union[Path, str]
        Directory path of HASC dataset, which is parent directory of "BasicActivity" directory.

    meta: pd.DataFrame
        meta data loaded by 'load_meta'.

    Returns
    -------
    data, meta: List[pd.DataFrame], pd.DataFrame
        Sensor data segmented by activity and subject.

    See Also
    --------
    The order of 'data' and 'meta' correspond.

    e.g. meta.iloc[0] is meta data of data[0].
    """
    path = check_path(path)

    raw = load_raw(path, meta)
    data, meta = reformat(raw)
    return data, meta

def load_meta(path:Path) -> pd.DataFrame:
    """Function for loading meta data of HASC dataset

    Parameters
    ----------
    path: Path
        Directory path of HASC dataset, which is parent directory of "BasicActivity" directory.

    Returns
    -------
    metas: pd.DataFrame:
        meta data of HASC dataset.
    """

    def replace_meta_str(s:str) -> str:
        s = s.replace('：', ':')
        s = s.replace('TerminalPosition:TerminalPosition:',
                            'TerminalPosition:')
        s = s.replace('sneakers:leathershoes', 'sneakers;leathershoes')
        s = s.replace('Frequency(Hz)', 'Frequency')
        return s

    def read_meta(file:Path):
        with file.open(mode='rU', encoding='utf-8') as f:
            ret = [s.strip() for s in f.readlines()]
            ret = filter(bool, ret)
            ret = [replace_meta_str(s) for s in ret]
            ret = [e.partition(':')[0::2] for e in ret]
            ret = {key.strip(): val.strip() for key, val in ret}
        act, person, file_name = file.parts[-3:]
        ret['act'] = act
        ret['person'] = person
        ret['file'] = file_name.split('.')[0]
        return ret

    path = path / 'BasicActivity'
    assert path.exists(), '{} is not exists.'.format(str(path))
    files = path.glob('**/**/*.meta')
    metas = [read_meta(p) for p in files]
    metas = functools.reduce(
        lambda a, b: a.append([b]),
        metas,
        pd.DataFrame()
    )
    return metas

def load_raw(path:Path, meta:Optional[pd.DataFrame]=None) -> Tuple[List[pd.DataFrame], pd.DataFrame]:
    """Function for loading raw data of HASC dataset

    Parameters
    ----------
    path: Path
        Directory path of HASC dataset, which is parent directory of "BasicActivity" directory.

    meta: pd.DataFrame
        meta data loaded by 'load_meta'.

    Returns
    -------
    data, meta: List[pd.DataFrame], pd.DataFrame
        raw data of HASC dataset.

        Each item in 'data' is a part of dataset, which is splited by subject.

    See Also
    --------
    The order of 'data' and 'meta' correspond.

    e.g. meta.iloc[0] is meta data of data[0].
    """

    def read_acc(path:Path) -> pd.DataFrame:
        if path.exists():
            try:
                ret = pd.read_csv(str(path), index_col=0, names=('x', 'y', 'z'))
                return ret
            except:
                print('[load] different data format:', str(path))
        else:
            print('[load] not found:', str(path))
        return pd.DataFrame()

    if meta is None:
        meta = load_meta(path)

    path = path / 'BasicActivity'
    data = [
        read_acc(path / row.act / row.person / '{}-acc.csv'.format(row.file))
        for row in meta.itertuples()
    ]

    return data, meta

def reformat(raw) -> Tuple[List[pd.DataFrame], pd.DataFrame]:
    """Function for reformating

    Parameters
    ----------
    raw:
        data loaded by 'load_raw'

    Returns
    -------
    data, meta: List[pd.DataFrame], pd.DataFrame
        Sensor data segmented by activity and subject.

    See Also
    --------
    The order of 'data' and 'meta' correspond.

    e.g. meta.iloc[0] is meta data of data[0].
    """
    data, meta = raw
    return data, meta


import math
import statistics
import warnings

import numpy as np
from hmmlearn.hmm import GaussianHMM
from sklearn.model_selection import KFold
from asl_utils import combine_sequences

class ModelSelector(object):
    '''
    base class for model selection (strategy design pattern)
    '''

    def __init__(self, all_word_sequences: dict, all_word_Xlengths: dict, this_word: str,
                 n_constant=3,
                 min_n_components=2, max_n_components=10,
                 random_state=14, verbose=False):
        self.words = all_word_sequences
        self.hwords = all_word_Xlengths
        self.sequences = all_word_sequences[this_word]
        self.X, self.lengths = all_word_Xlengths[this_word]
        self.this_word = this_word
        self.n_constant = n_constant
        self.min_n_components = min_n_components
        self.max_n_components = max_n_components
        self.random_state = random_state
        self.verbose = verbose

    def select(self):
        raise NotImplementedError

    def base_model(self, num_states):
        # with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        # warnings.filterwarnings("ignore", category=RuntimeWarning)
        try:
            hmm_model = GaussianHMM(n_components=num_states, covariance_type="diag", n_iter=1000,
                                    random_state=self.random_state, verbose=False).fit(self.X, self.lengths)
            if self.verbose:
                print("model created for {} with {} states".format(self.this_word, num_states))
            return hmm_model
        except:
            if self.verbose:
                print("failure on {} with {} states".format(self.this_word, num_states))
            return None

class SelectorConstant(ModelSelector):
    """ select the model with value self.n_constant

    """

    def select(self):
        """ select based on n_constant value

        :return: GaussianHMM object
        """
        best_num_components = self.n_constant
        return self.base_model(best_num_components)

class SelectorBIC(ModelSelector):
    """ select the model with the lowest Baysian Information Criterion(BIC) score

    http://www2.imm.dtu.dk/courses/02433/doc/ch6_slides.pdf
    Bayesian information criteria: BIC = -2 * logL + p * logN
    """

    def select(self):
        """ select the best model for self.this_word based on
        BIC score for n between self.min_n_components and self.max_n_components

        :return: GaussianHMM object
        """
        warnings.filterwarnings("ignore", category=DeprecationWarning)

        # initialize best score and model
        best_score = float("inf")
        best_model = None

        # loop through all the components
        for num_state in range(self.min_n_components, self.max_n_components):
            try:
                model = GaussianHMM(n_components=num_state, covariance_type="diag", n_iter=1000,
                                    random_state=self.random_state, verbose=False).fit(self.X, self.lengths)
                logL = model.score(self.X, self.lengths)

                # Using the BIC equations
                parameters = num_state * num_state + 2 * num_state * len(self.X[0]) - 1
                bic = (-2) * logL + math.log(len(self.X)) * parameters

                if bic < best_score:
                    best_score = bic
                    best_model = model
            except:
                pass
        return best_model if best_model is not None else self.base_model(self.n_constant)

class SelectorDIC(ModelSelector):
    ''' select best model based on Discriminative Information Criterion

    Biem, Alain. "A model selection criterion for classification: Application to hmm topology optimization."
    Document Analysis and Recognition, 2003. Proceedings. Seventh International Conference on. IEEE, 2003.
    http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.58.6208&rep=rep1&type=pdf
    DIC = log(P(X(i)) - 1/(M-1)SUM(log(P(X(all but i))
    '''

    def select(self):
        warnings.filterwarnings("ignore", category=DeprecationWarning)

         # TODO implement model selection based on DIC scores
        # initialize best score and model
        best_score = float("-inf")
        best_model = None

        # loop through all the components
        for num_state in range(self.min_n_components, self.max_n_components):
            try:
                model = GaussianHMM(n_components=num_state, covariance_type="diag", n_iter=1000,
                                    random_state=self.random_state, verbose=False).fit(self.X, self.lengths)
                logL = model.score(self.X, self.lengths)

                total_logL = 0.0
                M = 0
                for word in self.hwords:
                    if word != self.this_word:
                        X, lengths = self.hwords[word]
                        total_logL += model.score(X, lengths)
                        M += 1

                dic = logL - total_logL / M

                if dic > best_score:
                    best_score = dic
                    best_model = model
            except:
                pass
        return best_model if best_model is not None else self.base_model(self.n_constant)

class SelectorCV(ModelSelector):
    ''' select best model based on average log Likelihood of cross-validation folds

    '''

    def select(self):
        warnings.filterwarnings("ignore", category=DeprecationWarning)

        # initialize best score and model
        best_score = float("-inf")
        best_model = None
        logL_list = []

        # if there is not enough samples to do kfold split
        if (len(self.sequences) < 2):
            return self.base_model(self.n_constant)

        # loop through all the components
        split_method = KFold(n_splits=min(3,len(self.sequences)))
        for num_state in range(self.min_n_components, self.max_n_components):
            # split the word sequences in to training and testing set
            # using KFold split
            try:
                for train_idx, test_idx in split_method.split(self.sequences):
                    X_train, lengths_train = combine_sequences(train_idx, self.sequences)
                    model = GaussianHMM(n_components=num_state, covariance_type="diag", n_iter=1000,
                                        random_state=self.random_state, verbose=False).fit(X_train, lengths_train)
                    X_test, lengths_test = combine_sequences(test_idx, self.sequences)
                    logL = model.score(X_test, lengths_test)
                    logL_list.append(logL)

                mean_score = np.mean(logL_list)

                if mean_score > best_score:
                    best_score = mean_score
                    best_model = model
            except:
                pass

        return best_model if best_model is not None else self.base_model(self.n_constant)


# -*- coding: utf-8 -*-
"""
Created on Wed Mar  7 21:32:59 2018

@author: ting
"""
import time
import numpy as np
import pandas as pd
import datetime
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM,TimeDistributed
from keras.callbacks import LearningRateScheduler, EarlyStopping
from keras.callbacks import ModelCheckpoint
from keras import optimizers
import os
#np.seterr(divide='ignore', invalid='ignore')

t0 = time.time()

column = ['Timestamp','Station_ID','Freeway','Direction_Travel',
           'Station_Type','Samples','Observed','Total_Flow',
           'Av_Occupancy','Av_Speed']
#dfaa_freeway = pd.read_csv('C:/Users/ting/Desktop/Freeway/Freeways-Rawar/Freeways-Rawaa.csv', engine='python')
#print(dfaa_freeway)
df_a = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minaa.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],) #不會讀第一行'
df_b = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minab.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_c = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minac.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_d = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minad.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_e = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minae.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_f = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minaf.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_g = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minag.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df_h = pd.read_csv("D:/San Diego Dataset/San Diego Dataset/Freeways-5Min/Freeways-5Minah.csv",header=None, skipinitialspace=True, usecols=[0,1,2,3,4,5,6,7,8,9],)
df.columns = column
df = df.dropna(axis=0, subset=['Av_Speed','Av_Occupancy'])

RESERVE_FEATURE= ['Timestamp','Station_ID','Total_Flow', 'Av_Occupancy','Av_Speed']
df= df[RESERVE_FEATURE]

#c0 = df_time.groupby(by=[df_time.index.day,df_time.index.hour])[FEATURE].count()
#sum0 = df_time.groupby(by=[df_time.index.day,df_time.index.hour])[FEATURE].sum()
#mean0 = sum0/c0
#
#fig, ax = plt.subplots(figsize=(15,7))
#mean0.plot(ax=ax)
#plt.xticks(list(mean0.index))
#plt.show()
#plt.close()

df.sort_values(by = ['Timestamp'])
df['Timestamp'] = pd.to_datetime(df['Timestamp'],format="%m/%d/%Y %H:%M:%S") # dataframe tp datetime
mask = (df['Timestamp'] > '1/1/2010')
df = df.loc[mask]

def set_location(dataset , location):

    df_sta = dataset.loc[dataset['Station_ID']==location]
    df_sta.sort_values(by = ['Timestamp'])
    df_sta['Timestamp'] = pd.to_datetime(df_sta['Timestamp'],format="%m/%d/%Y %H:%M:%S") # dataframe tp datetime
    df_sta['weekday'] = df_sta['Timestamp'].dt.dayofweek  #dayweek = 0-6  sereis.dt.dayofweek
    df_sta['weekday'] = df_sta['weekday'].apply(lambda x:  0 if x<5  else 1)
    df_sta['min_index'] = df_sta['Timestamp'].apply(lambda d: (d.hour*60 + d.minute) / 5)
    #mask = (df_sta['Timestamp'] > '2/15/2010') & (df_sta['Timestamp'] <= '2/23/2010')
    mask = (df_sta['Timestamp'] > '1/1/2010')
    df_sta = df_sta.loc[mask]
    df_sta.index = df_sta['Timestamp']
    df_sta['Av_Speed_t1']= df_sta['Av_Speed'].shift(-1)
    df_sta['Av_Speed_t2']= df_sta['Av_Speed'].shift(-2)
    df_sta['Av_Speed_t3']= df_sta['Av_Speed'].shift(-3)
    df_sta['Av_Speed_t4']= df_sta['Av_Speed'].shift(-4)
    df_sta['Av_Speed_t5']= df_sta['Av_Speed'].shift(-5)
    return df_sta

df_sta2 = set_location(df,1108678)
df_sta1 = set_location(df,1108659)
df_sta3 = set_location(df,1119921)
df_sta4 = set_location(df,1119928)
df_sta5 = set_location(df,1119934)

df_combineLoc= pd.concat([df_sta1, df_sta2], axis=1,join_axes=[df_sta2.index])  #以第一張dataframe為index ，若合併後有缺就是NAN

df_combineLoc.columns = ['Timestamp','Station_ID','Total_Flow','Av_Occupancy','Av_Speed','weekday','min_index',
                         'Av_Speed_t1','Av_Speed_t2','Av_Speed_t3','Av_Speed_t4','Av_Speed_t5',
                         'Timestamp2','Station_ID2','Total_Flow2','Av_Occupancy2','Av_Speed2','weekday2','min_index2']
df_combineLoc = df_combineLoc.drop(['Timestamp2','Station_ID2','weekday2','min_index2'], 1)

#僅測試用看是否合併後還有NA存在
df_combineLoc = df_combineLoc.fillna(method='ffill')
df_combineLoc.isnull().sum()
#df_combineLoc.isnull().values.any()
#--------------------------------------------------------------------------

TIMESTEP = 5
LAG = 6
output_unit = 5
TRAIN_FEATURE = ['Av_Speed','Av_Occupancy','weekday','min_index',
                 'Av_Speed_t1','Av_Speed_t2','Av_Speed_t3','Av_Speed_t4','Av_Speed_t5']
#TRAIN_FEATURE = ['Av_Speed','Av_Occupancy','Av_Speed2','Av_Occupancy2','weekday','min_index']
LABEL_FEATURE = 'Av_Speed'
#def supervised(dataset, lag=1):
#    df = pd.DataFrame(dataset)
#    df_shift = df.shift()
#    sumdf = pd.concat([df_shift,df], axis=1)
#    sumdf.fillna(0, inplace=True)
#    sumdf = np.array(sumdf)
#    return sumdf

def multi_supervised(dataset):
    df_shift = dataset[LABEL_FEATURE].shift(-LAG)
    sumdf = pd.concat([dataset[TRAIN_FEATURE],df_shift], axis=1)
    sumdf.fillna(0, inplace=True)
#    result= sumdf.values
#    sumdf = np.array(sumdf)
    return sumdf

raw = multi_supervised(df_sta2)
#raw = multi_supervised(df_combineLoc)
raw= raw.values

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(raw[:,:-1], raw[:,-1], test_size = 0.2 ,shuffle = False, stratify = None)   #shuffle 不要洗牌(因為時序資料要依序列出) stratify train set/ test set 當中的比例相同
#from sklearn.preprocessing import StandardScaler
#scaled_features = StandardScaler().fit_transform(df.values)
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler()
#X_train_scaled = np.reshape(X_train,(-1,1))  #先變成 N*1

X_train_scaled = sc.fit_transform(X_train)
y_train_scaled = np.reshape(y_train,(-1,1))
y_train_scaled = sc.fit_transform(y_train_scaled)
X_train_scaled = np.reshape(X_train_scaled, (-1,TIMESTEP,len(TRAIN_FEATURE))) #未知列數,5代表timestep,1代表維度
y_train_scaled = np.reshape(y_train_scaled, (-1,output_unit))

#X_test_scaled = np.reshape(X_test,(-1,1))
X_test_scaled = sc.transform(X_test)
X_test_scaled = np.reshape(X_test_scaled, (-1, TIMESTEP, len(TRAIN_FEATURE)))

#-------------------------------------------------------------------------------
save_path = "./traffic_modelsave/"+time.strftime('%m%d%H%M')   #save的資料夾
os.mkdir(save_path)

#training model

history = []
patience=30
epochs =600
batch_size = 32
hidden_units = 8

earlystop = EarlyStopping(patience = patience)
modelsave = ModelCheckpoint(filepath= save_path+'/model.h5', save_best_only=True, verbose=1)
#opt = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
regressor = Sequential()
regressor.add(LSTM(units = hidden_units,
                   activation = 'relu',
                   return_sequences=True,  #有這行就是多對多
#                   stateful=stateful,
#                   dropout=0.2,
                   input_shape = (
                                  X_train_scaled.shape[1],
                                  X_train_scaled.shape[2])))

regressor.add(LSTM(units = hidden_units,
                   activation = 'relu',
#                   stateful=stateful,
#                   return_sequences=True,
#                   dropout=0.2,
                   ))
#regressor.add(TimeDistributed(Dense(TIMESTEP)))
regressor.add(Dense(units = output_unit)) #最後一層要輸出幾個

regressor.compile(optimizer = 'adam',
                  loss = 'mean_squared_error')
his= regressor.fit(X_train_scaled,
              y_train_scaled,
              batch_size = batch_size,
              epochs = epochs,
              verbose = 1,
              validation_split=0.2,
              callbacks=[earlystop,modelsave])

history.append(his)
regressor.summary()

#-------------------------------------------------------------------------------------------------
#print the result
train_pred = regressor.predict(X_train_scaled)
train_pred = sc.inverse_transform(train_pred)
train_pred = train_pred.reshape(-1)

y_pred = regressor.predict(X_test_scaled)
y_pred = sc.inverse_transform(y_pred)
y_pred = y_pred.reshape(-1)

#print the evaluation
from sklearn.metrics import mean_squared_error,mean_absolute_error

def mean_absolute_percentage_error(y_true, y_pred):
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

train_mse = mean_squared_error(y_train, train_pred)
print("Train_mse",train_mse)
train_mae= mean_absolute_error(y_train, train_pred)
print("Train_mae=",train_mae)
train_mape = mean_absolute_percentage_error(y_train, train_pred)
print("Train_ MAPE:",train_mape)

test_mse = mean_squared_error(y_test, y_pred)
print("Test_mse",test_mse)
test_mae= mean_absolute_error(y_test, y_pred)
print("Test_mae=",test_mae)
test_mape = mean_absolute_percentage_error(y_test, y_pred)
print("Test_ MAPE:",test_mape)

#write log file
with open(save_path + './Log.txt','w') as fh:
    regressor.summary(print_fn = lambda x: fh.write(x + '\n'))
    fh.write("TIMESTEP:"+str(TIMESTEP)+
             "\ndimension:"+str(len(TRAIN_FEATURE))+
             "\nTRAIN_FEATURE:"+str(TRAIN_FEATURE)+
             "\nlag:"+str(LAG)+
             "\nunits:"+str(hidden_units)+
             "\noutput_units:"+str(output_unit )+
             "\nbatch_size:"+str(batch_size)+
             "\npatience:"+str(patience)+
             "\nepochs:"+str(epochs)+
             "\nTrain_mse="+str(train_mse)+
             "\nTrain_mae="+str(train_mae)+
             "\nTrain_mape="+str(train_mape)+
             "\ntest_mse="+str(test_mse)+
             "\ntest_mae="+str(test_mae)+
             "\ntest_mape="+str(test_mape)
             )

#draw the ground truth and prediction
def show_predict(y_pred,y_test):
    plt.plot(y_pred)
    plt.plot(y_test)
    plt.ylabel("speed")
    plt.xlabel("time")
    plt.xlim(0,100)
    plt.ylim(0, 80)
    plt.legend(['y_pred', 'y_test'], loc='best')
    plt.savefig(save_path+'/speed.png')
    plt.show()
    plt.close()

show_predict(y_pred,y_test)

#draw the loss function
def show_train_history(i,train_history):
    plt.plot(train_history.history['loss'])
    plt.plot(train_history.history['val_loss'])
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.xlim(0, 600)
    plt.ylim(0, 0.005)
    plt.legend(['train_loss', 'val_loss'], loc='best')
    plt.savefig(save_path+'/loss.png')
    plt.show()
    plt.close()

for i in range(len(history)):
    show_train_history(i,history[i])

#os.makedirs("./traffic/traffic_modelsave/"+time.strftime('%m%d%H%M'))
#text_file = open(save_path+"./Log.txt", "w")
#text_file.write("TIMESTEP:"+str(TIMESTEP)+"\ndimension:"+str(len(TRAIN_FEATURE))+
#                "\nlag:"+str(LAG)+
#                "\nTrain_mae="+str(train_mae)+"\ntrain_mape="+str(train_mape)+
#                "\ntest_mae="+str(test_mae)+"\ntest_mape="+str(test_mape)+
#                "\nstateful = "+str(stateful))


import os,sys,subprocess,time
import argparse
import numpy as np

###################################################################
# Check the python version and import configparser
###################################################################
if sys.version_info[0]==2:
        import ConfigParser
        config = ConfigParser.RawConfigParser()
elif sys.version_info[0]==3:
        import configparser
        config = configparser.ConfigParser()
###############################################################################
#You can modify any options in the parameters.ini file by the command terminal
###############################################################################
parser = argparse.ArgumentParser()
parser.add_argument('--spart', action = 'store', dest = 'spart', default = 1        ,   help = 'starting part')
parser.add_argument('--epart', action = 'store', dest = 'epart', default = 5        , help = 'end part')
parser.add_argument('--user' , action = 'store', dest = 'user' , default = "amarins",  help = '')
###############################################################################
#Variables
###############################################################################
arguments = parser.parse_args()
spart = int(arguments.spart)
epart = int(arguments.epart)
user  = int(arguments.user)
###############################################################################
###############################################################################

def reportERROR(jobNAME=None, err=None, text=None):
        import file_verification as ver
        path      = path  = "/".join(("/share/data1",user,"PS1zxcorr"))
        directory = "ERRORS-jobs"
        filename  = "".join(("error",jobNAME,".err"))
        ver.file_verification(path,filename,directory)
        f = open(path,"w+")
        f.write(err)
        if text!=None:
                text = str(text)
                f.write(text)
        f.close()
def JOBID(USER="amarins", jobNAME = None): #recebe o nome de um job (sbatch) rodando no SPLINTER e devolve o seu jobID.
        qstat_out, err = subprocess.Popen("qstat", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
        qstat_out      = qstat_out.split("\n")
        qstat_out      = qstat_out[2:-1]
        jobID          = []

        if len(err)==0:
                for i in range(len(qstat_out)):
                        qstat = qstat_out[i].split()
                        user  = qstat[2]
                        jobname = qstat[1]
                        if user == USER and jobname == jobNAME:
                                print(user,jobname)
                                jobID.append(qstat[0])
                return jobID
        else:
                reportERROR(jobNAME,err,None)
                return None

###############################################################################
###############################################################################
jobs = np.arange(start=spart,stop=epart+1,step=1,dtype=np.int)
label = "sbatch"
path  = "/".join(("/share/data1",user,"PS1zxcorr/sbatch_files/"))
for i in jobs:
        job_sbatch = ".".join(("_".join((label,str(i))),"sh"))
        if i==jobs[0]:
                        jobNAME = "_".join(("PS1",str(i)))
                        cmd     = " ".join(("sbatch",job_sbatch))
                        print(cmd)
                        job, err_job = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
                        time.sleep(1)#2sec
                        ID      = JOBID(USER=user, jobNAME=jobNAME)
                        ID      = int(ID[0])
                        print(ID)
        else:
                        jobNAME = "_".join(("PS1",str(i)))
                        cmd         = " ".join(("sbatch", "".join(("--depend=afterany:",str(ID))),job_sbatch))
                        print(cmd)
                        job, err_job = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
                        time.sleep(1)#2sec
                        ID      = JOBID(USER=user, jobNAME=jobNAME)
                        ID      = int(ID[0])
                        print(ID)


import csv
import datetime
from model import User, Trade, Order, db

def find_orders(file_path):
    orders_list=[]
    with open(file_path) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        account_trade_history = False

        for row in csv_reader:

            if len(row) > 0 and account_trade_history == False:
                #Finds where Account Trade History section begins
                if "Account Trade History" in row[0]:
                    account_trade_history = True
                    trade_counter = 0

                    continue
            #reverts boolean after end of trade history is reached
            elif len(row) == 0 and account_trade_history == True:
                account_trade_history = False

            #Ignore rows that aren't in Account Trade History section
            if account_trade_history:
                #Skips first row of trade history, which is column headers
                if trade_counter > 0:
                    orders_list.append(row)
                trade_counter += 1

    return orders_list

def parse_order_row(order_row):
    exec_time = datetime.datetime.strptime(order_row[1], "%m/%d/%y %H:%M:%S")
    spread = order_row[2]
    side = order_row[3]
    qty = int(order_row[4])
    pos_effect = order_row[5]
    symbol = order_row[6]

    #expiration date is only for options and future contracts
    #future contracts only have the month and year because it always expires on the third friday of the month
    if order_row[7] == '':
        expiration = None
    elif len(order_row[7].split()) == 2:
        expiration = datetime.datetime.strptime(order_row[7], "%b %y")
    elif len(order_row[7].split()) == 3:
        expiration = datetime.datetime.strptime(order_row[7], "%d %b %y")

    #strike price is for only options
    if order_row[8] == '':
        strike_price = None
    else:
        strike_price = float(order_row[8])

    type = order_row[9]
    price = float(order_row[10])
    order_type = order_row[12]

    return exec_time, spread, side, qty, pos_effect, symbol, expiration, strike_price, type, price, order_type

def create_order(user_id, exec_time, spread, side, qty, pos_effect, symbol, expiration, strike_price, type, price, order_type):

    #query database to find open trade for the user that matches the order symbol, spread, & side
    open_trade = Trade.query.filter_by(user_id=user_id, symbol=symbol, spread=spread, open=True).first()

    if open_trade is None:

        #make a new trade if no open trade is found
        open_trade = Trade(user_id=user_id, open_date=exec_time, symbol=symbol, entry_price=price, side=side, spread=spread, position=qty)
        db.session.add(open_trade)
        db.session.commit()

        order = add_order_to_db(trade=open_trade, exec_time=exec_time, spread=spread, side=side, qty=qty, pos_effect=pos_effect, symbol=symbol, expiration=expiration, strike_price=strike_price, type=type, price=price, order_type=order_type)

    else:

        #check if new order qty is large enough to close the current open position and one is positive while the other is negative
        #split order into two separate orders, one that closes the open trade, one that opens a new trade
        if abs(qty) > abs(open_trade.position) and ((open_trade.position ^ qty) < 0):

            #split qty's apart
            qty_one = open_trade.position * -1
            qty_two = qty - qty_one

            #closing order
            order_one = add_order_to_db(trade=open_trade, exec_time=exec_time, spread=spread, side=side, qty=qty_one, pos_effect=pos_effect, symbol=symbol, expiration=expiration, strike_price=strike_price, type=type, price=price, order_type=order_type)

            close_trade(trade=open_trade, closing_order=order_one)

            #new opening trade
            new_open_trade = Trade(user_id=user_id, open_date=exec_time, symbol=symbol, entry_price=price, side=side, spread=spread, position=qty_two)
            db.session.add(new_open_trade)
            db.session.commit()

            #new opening order
            order_two = add_order_to_db(trade=new_open_trade, exec_time=exec_time, spread=spread, side=side, qty=qty_two, pos_effect=pos_effect, symbol=symbol, expiration=expiration, strike_price=strike_price, type=type, price=price, order_type=order_type)

        else:
            #add order to open trade
            order = add_order_to_db(trade=open_trade, exec_time=exec_time, spread=spread, side=side, qty=qty, pos_effect=pos_effect, symbol=symbol, expiration=expiration, strike_price=strike_price, type=type, price=price, order_type=order_type)

            open_trade.position += qty
            db.session.commit()

            #if trade position equals zero, consider the trade closed
            if open_trade.position == 0:
                close_trade(trade=open_trade, closing_order=order)

def add_order_to_db(trade, exec_time, spread, side, qty, pos_effect, symbol, expiration, strike_price, type, price, order_type):
        order = Order(trade_id=trade.trade_id, exec_time=exec_time, spread=spread, side=side, qty=qty, pos_effect=pos_effect, symbol=symbol, expiration=expiration, strike_price=strike_price, type=type, price=price, order_type=order_type)
        db.session.add(order)
        calc_total_trade_cost(spread=spread, qty=qty, price=price, trade=trade)
        db.session.commit()
        return order

def calc_total_trade_cost(spread, qty, price, trade):

    if spread == "SINGLE":
        trade.return_amt += -(100*qty*price)
    elif spread == "STOCK":
        trade.return_amt += -(qty*price)
    elif spread == "FUTURE":
        trade.return_amt += -(5*qty*price)

def close_trade(trade, closing_order):
    trade.position = 0
    trade.exit_price = closing_order.price
    trade.open = False
    db.session.commit()
    if trade.return_amt > 0:
        trade.result = "WIN"
    elif trade.return_amt == 0:
        trade.result = "EVEN"
    else:
        trade.result = "LOSS"
    db.session.commit()