import json
import logging
import requests

class EmonCMS:

    def __init__(self, api_base_uri, api_key, json_variant='json'):
        self.name = 'EmonCMS'
        self._base_uri = api_base_uri + '/input/post.json'
        self._api_key = api_key
        self._json_variant = json_variant
        self._param_names = {}

    def write(self, data):
        for device, values in data:
            if not values:
                continue
            if not device.name in self._param_names:
                self._param_names[device.name] = [
                        x.name for x in device.params()]
            sample_as_json = json.dumps(dict(zip(
                    self._param_names[device.name], values)))
            logging.debug('Sending "%s" to EmonCMS...', sample_as_json)
            response = requests.post(self._base_uri, data={
                'node': device.name,
                'apikey': self._api_key,
                self._json_variant: sample_as_json})
            logging.debug('EmonCMS responded %s - %s',
                    response.status_code, response.reason)
            if response.reason != 'OK':
                logging.warning('Unexpected EmonCMS response: %s - %s',
                        response.status_code, response.reason)


from basketball_reference_scraper.teams import get_roster, get_team_stats, get_opp_stats, get_roster_stats, get_team_misc
from basketball_reference_scraper.seasons import get_schedule
from basketball_reference_scraper.box_scores import get_box_scores
from basketball_reference_scraper.injury_report import get_injury_report
from utils import *
import sys
import pandas as pd
import numpy as np
import datetime
pd.options.mode.chained_assignment = None

# TODO:
#   - Get injury report
#   - Check if away team has anyone on the report
#   - Check if home team has anyone on the report
#   - If so, get the player and return his win shares
#   - Add up total missing win shares and create a column displaying that
#   - Standardize it

def convert_team_names(entire_schedule):
    team_abbreviations = {'Atlanta Hawks' : 'ATL',
                        'Brooklyn Nets' : 'BRK',
                        'Boston Celtics' : 'BOS',
                        'Charlotte Hornets' : 'CHO',
                        'Chicago Bulls' : 'CHI',
                        'Cleveland Cavaliers' : 'CLE',
                        'Dallas Mavericks' : 'DAL',
                        'Denver Nuggets' : 'DEN',
                        'Detroit Pistons' : 'DET',
                        'Golden State Warriors' : 'GSW',
                        'Houston Rockets' : 'HOU',
                        'Indiana Pacers' : 'IND',
                        'Los Angeles Clippers' : 'LAC',
                        'Los Angeles Lakers' : 'LAL',
                        'Memphis Grizzlies' : 'MEM',
                        'Miami Heat' : 'MIA',
                        'Milwaukee Bucks' : 'MIL',
                        'Minnesota Timberwolves' : 'MIN',
                        'New Orleans Pelicans' : 'NOP',
                        'New York Knicks' : 'NYK',
                        'Oklahoma City Thunder' : 'OKC',
                        'Orlando Magic' : 'ORL',
                        'Philadelphia 76ers' : 'PHI',
                        'Phoenix Suns' : 'PHO',
                        'Portland Trail Blazers' : 'POR',
                        'Sacramento Kings' : 'SAC',
                        'San Antonio Spurs' : 'SAS',
                        'Toronto Raptors' : 'TOR',
                        'Utah Jazz' : 'UTA',
                        'Washington Wizards' : 'WAS'}

    team_abbreviations_keys = list(team_abbreviations.keys())

    row = 0
    for home_team in entire_schedule['HOME']:
        for i in range(0, 30):
            if team_abbreviations_keys[i] in home_team:
                entire_schedule.at[row, 'HOME'] = team_abbreviations[team_abbreviations_keys[i]]
        row += 1
    row = 0
    for away_team in entire_schedule['VISITOR']:
        for i in range(0, 30):
            if team_abbreviations_keys[i] in away_team:
                entire_schedule.at[row, 'VISITOR'] = team_abbreviations[team_abbreviations_keys[i]]
        row += 1

    a = datetime.datetime(2020, 7, 30)
    b = datetime.datetime(2020, 7, 31)
    c = datetime.datetime(2020, 8, 1)
    d = datetime.datetime(2020, 8, 2)
    e = datetime.datetime(2020, 8, 3)
    f = datetime.datetime(2020, 8, 4)
    g = datetime.datetime(2020, 8, 5)
    h = datetime.datetime(2020, 8, 6)
    i = datetime.datetime(2020, 8, 7)
    j = datetime.datetime(2020, 8, 8)
    k = datetime.datetime(2020, 8, 9)
    l = datetime.datetime(2020, 8, 10)
    m = datetime.datetime(2020, 8, 11)
    n = datetime.datetime(2020, 8, 12)
    o = datetime.datetime(2020, 8, 13)
    p = datetime.datetime(2020, 8, 14)

    date = [a,a,b,b,b,b,b,b,c,c,c,c,c,d,d,d,d,d,d,e,e,e,e,e,e,f,f,f,f,f,f,g,g,g,g,g,g,h,h,h,h,h,h,i,i,i,i,i,i,j,j,j,j,j,k,k,k,k,k,k,k,l,l,l,l,l,m,m,m,m,m,m,m,n,n,n,n,o,o,o,o,o,o,o,p,p,p,p]
    #date = ['7/30/2020', '7/30/2020', '7/31/2020', '7/31/2020', '7/31/2020', '7/31/2020', '7/31/2020', '7/31/2020', '8/1/2020', '8/1/2020', '8/1/2020', '8/1/2020', '8/1/2020', '8/2/2020', '8/2/2020', '8/2/2020', '8/2/2020', '8/2/2020', '8/2/2020', '8/3/2020', '8/3/2020', '8/3/2020', '8/3/2020', '8/3/2020', '8/3/2020', '8/4/2020', '8/4/2020', '8/4/2020', '8/4/2020', '8/4/2020', '8/4/2020', '8/5/2020', '8/5/2020', '8/5/2020', '8/5/2020', '8/5/2020', '8/5/2020', '8/6/2020', '8/6/2020', '8/6/2020', '8/6/2020', '8/6/2020', '8/6/2020', '8/7/2020', '8/7/2020', '8/7/2020', '8/7/2020', '8/7/2020', '8/7/2020', '8/8/2020', '8/8/2020', '8/8/2020', '8/8/2020', '8/8/2020',
    #                            '8/9/2020', '8/9/2020', '8/9/2020', '8/9/2020', '8/9/2020', '8/9/2020', '8/9/2020', '8/10/2020', '8/10/2020', '8/10/2020', '8/10/2020', '8/10/2020', '8/11/2020', '8/11/2020', '8/11/2020', '8/11/2020', '8/11/2020', '8/11/2020', '8/11/2020', '8/12/2020', '8/12/2020', '8/12/2020', '8/12/2020', '8/13/2020', '8/13/2020', '8/13/2020', '8/13/2020', '8/13/2020', '8/13/2020', '8/13/2020', '8/14/2020', '8/14/2020', '8/14/2020', '8/14/2020']

    empty = [" "] * len(date)

    df1 = pd.DataFrame({'VISITOR' : ['UTA', 'LAC', 'ORL', 'MEM', 'PHO', 'BOS', 'SAC', 'HOU', 'MIA', 'UTA', 'NOP', 'PHI', 'LAL', 'WAS', 'POR', 'SAS', 'SAC', 'MIL', 'DAL', 'TOR', 'DEN', 'IND', 'MEM', 'SAS', 'LAL', 'BRK', 'DAL', 'PHO', 'ORL', 'BOS', 'HOU', 'MEM', 'PHI', 'DEN', 'OKC', 'TOR', 'BRK', 'NOP', 'MIA', 'IND', 'LAC', 'POR', 'LAL', 'UTA', 'OKC', 'SAC', 'ORL', 'WAS', 'BOS', 'LAC', 'UTA', 'LAL', 'PHO', 'MIL', 'WAS', 'MEM', 'SAS', 'ORL', 'PHI', 'HOU', 'BRK', 'OKC', 'DAL', 'TOR', 'IND', 'DEN', 'BRK', 'HOU', 'PHO', 'POR', 'BOS', 'NOP', 'MIL', 'IND', 'TOR', 'MIA', 'LAC', 'WAS', 'POR', 'SAC', 'MIL', 'NOP', 'DAL', 'SAS', 'PHI', 'MIA', 'OKC', 'DEN'],
                       'VISITOR_PTS': empty,
                       'HOME' : ['NOP', 'LAL', 'BRK', 'POR', 'WAS', 'MIL', 'SAS', 'DAL', 'DEN', 'OKC', 'LAC', 'IND', 'TOR', 'BRK', 'BOS', 'MEM', 'ORL', 'HOU', 'PHO', 'MIA', 'OKC', 'WAS', 'NOP', 'PHI', 'UTA', 'MIL', 'SAC', 'LAC', 'IND', 'MIA', 'POR', 'UTA', 'WAS', 'SAS', 'LAL', 'ORL', 'BOS', 'SAC', 'MIL', 'PHO', 'DAL', 'DEN', 'HOU', 'SAS', 'MEM', 'BRK', 'PHI', 'NOP', 'TOR', 'POR', 'DEN', 'IND', 'MIA', 'DAL', 'OKC', 'TOR', 'NOP', 'BOS', 'POR', 'SAC', 'LAC', 'PHO', 'UTA', 'MIL', 'MIA', 'LAL', 'ORL', 'SAS', 'PHI', 'DAL', 'MEM', 'SAC', 'WAS', 'HOU', 'PHI', 'OKC', 'DEN', 'BOS', 'BRK', 'LAL', 'MEM', 'ORL', 'PHO', 'UTA', 'HOU', 'IND', 'LAC', 'TOR'],
                       'HOME_PTS': empty,
                       'DATE' : date})

    entire_schedule = pd.concat([entire_schedule, df1], sort=False)

    return entire_schedule

def add_winner_column(entire_schedule):
    winner = []
    for index, row in entire_schedule.iterrows():
        if pd.isna(row['HOME_PTS']) or pd.isna(row['VISITOR_PTS']):
            winner.insert(index, float('Nan'))
        elif row['HOME_PTS'] > row['VISITOR_PTS']:
            winner.insert(index, 1)
        else:
            winner.insert(index, 0)
    entire_schedule["WINNER"] = winner
    return entire_schedule

def add_conference_game(entire_schedule):
    west_teams = [
        'DAL','DEN','GSW','HOU','LAC','LAL','MEM','MIN',
        'NOP','OKC','PHO','POR','SAC','SAS','UTA']
    east_teams = [
        'ATL','BRK','BOS','CHO','CHI','CLE','DET','IND',
        'MIA','MIL','NYK','ORL','PHI','TOR','WAS']
    conference_game = []
    for index, row in entire_schedule.iterrows():
        if (row['HOME'] in west_teams and row['VISITOR'] in west_teams) or (row['HOME'] in east_teams and row['VISITOR'] in east_teams):
            conference_game.insert(index, 1)
        else:
            conference_game.insert(index, 0)
    entire_schedule["CON_GAME"] = conference_game
    return entire_schedule

def add_team_stats(entire_schedule):
    team_misc_2020 = getTeamMisc(2020)
    team_misc_2019 = getTeamMisc(2019)
    last_year_win_percentage_home = []
    last_year_win_percentage_visitor = []
    count = 1
    for index, row in entire_schedule.iterrows():
        home = team_misc_2019.loc[team_misc_2019['TEAM'] == row['HOME'], 'W'].values[0]
        visitor = team_misc_2019.loc[team_misc_2019['TEAM'] == row['VISITOR'], 'W'].values[0]

        last_year_win_percentage_home.insert(index, home/82)
        last_year_win_percentage_visitor.insert(index, visitor/82)

        home_team_season_stats = team_misc_2020[team_misc_2020['TEAM'] == row['HOME']].add_prefix("HOME_")
        visitor_team_season_stats = team_misc_2020[team_misc_2020['TEAM'] == row['VISITOR']].add_prefix("VISITOR_")

        home_team_season_stats = home_team_season_stats[['HOME_NRtg', 'HOME_DRtg', 'HOME_ORtg', 'HOME_DRB%', 'HOME_SRS', 'HOME_MOV',
            'HOME_PACE']]
        visitor_team_season_stats = visitor_team_season_stats[['VISITOR_NRtg', 'VISITOR_DRtg', 'VISITOR_ORtg', 'VISITOR_DRB%', 'VISITOR_SRS', 'VISITOR_MOV',
            'VISITOR_PACE']]

        if count == 1:
            home_team_stats = home_team_season_stats
            visitor_team_stats = visitor_team_season_stats
        else:
            home_team_stats = pd.concat([home_team_stats, home_team_season_stats])
            visitor_team_stats = pd.concat([visitor_team_stats, visitor_team_season_stats])
        count += 1

    entire_schedule = entire_schedule.reset_index(drop=True)
    home_team_stats = home_team_stats.reset_index(drop=True)
    visitor_team_stats = visitor_team_stats.reset_index(drop=True)
    entire_schedule = pd.concat([entire_schedule, home_team_stats, visitor_team_stats], axis=1, sort=True)
    entire_schedule["HOME_LAST_SEASON_W%"] = last_year_win_percentage_home
    entire_schedule["VISITOR_LAST_SEASON_W%"] = last_year_win_percentage_visitor
    entire_schedule["HOME_GAME"] = [1] * len(entire_schedule)
    entire_schedule["AWAY GAME"] = [0] * len(entire_schedule)
    return entire_schedule

def add_win_percentage(entire_schedule):
    teams = [
        'ATL','BRK','BOS','CHO','CHI','CLE','DAL','DEN',
        'DET','GSW','HOU','IND','LAC','LAL','MEM','MIA',
        'MIL','MIN','NOP','NYK','OKC','ORL','PHI','PHO',
        'POR','SAC','SAS','TOR','UTA','WAS']
    init = [0] * 30
    data = {'wins' : init, 'losses' : init}

    injury_report = get_injury_report()
    win_loss = pd.DataFrame(data, index=teams)
    home_win_percentage = []
    visitor_win_percentage = []
    for index, row in entire_schedule.iterrows():

        if (win_loss.loc[row['HOME'], 'wins'] + win_loss.loc[row['HOME'], 'losses']) == 0:
            if win_loss.loc[row['HOME'], 'wins'] > 0:
                home_win_per = 1
            else:
                home_win_per = 0
        else:
            home_wins = win_loss.loc[row['HOME'], 'wins']
            if get_first_no_result_date(entire_schedule) == row['DATE']:
                injury_account = get_missing_win_shares(injury_report, row['HOME'])
                home_wins += injury_account
            home_win_per = home_wins / (win_loss.loc[row['HOME'], 'wins'] + win_loss.loc[row['HOME'], 'losses'])

        if (win_loss.loc[row['VISITOR'], 'wins'] + win_loss.loc[row['VISITOR'], 'losses']) == 0:
            if win_loss.loc[row['VISITOR'], 'wins'] > 0:
                visitor_win_per = 1
            else:
                visitor_win_per = 0
        else:
            visitor_wins = win_loss.loc[row['VISITOR'], 'wins']
            if get_first_no_result_date(entire_schedule) == row['DATE']:
                injury_account = get_missing_win_shares(injury_report, row['VISITOR'])
                visitor_wins += injury_account
            visitor_win_per = visitor_wins / (win_loss.loc[row['VISITOR'], 'wins'] + win_loss.loc[row['VISITOR'], 'losses'])

        home_win_percentage.insert(index, home_win_per)
        visitor_win_percentage.insert(index, visitor_win_per)

        if row['WINNER'] == 1:
            win_loss.at[row['HOME'],'wins'] = win_loss.loc[row['HOME'],'wins'] + 1
            win_loss.at[row['VISITOR'],'losses'] = win_loss.loc[row['VISITOR'],'losses'] + 1
        elif row['WINNER'] == 0:
            win_loss.at[row['VISITOR'],'wins'] = win_loss.loc[row['VISITOR'],'wins'] + 1
            win_loss.at[row['HOME'],'losses'] = win_loss.loc[row['HOME'],'losses'] + 1

    entire_schedule["HOME_W%"] = home_win_percentage
    entire_schedule["VISITOR_W%"] = visitor_win_percentage
    return entire_schedule

def add_home_away_splits(entire_schedule):
    teams = [
        'ATL','BRK','BOS','CHO','CHI','CLE','DAL','DEN',
        'DET','GSW','HOU','IND','LAC','LAL','MEM','MIA',
        'MIL','MIN','NOP','NYK','OKC','ORL','PHI','PHO',
        'POR','SAC','SAS','TOR','UTA','WAS']
    init = [0] * 30
    data = {'home_wins' : init, 'home_losses' : init, 'road_wins': init, 'road_losses' : init}
    home_road_split = pd.DataFrame(data, index=teams)
    home_win_at_home = []
    visitor_win_on_road = []
    for index, row in entire_schedule.iterrows():
        if (home_road_split.loc[row['HOME'], 'home_wins'] + home_road_split.loc[row['HOME'], 'home_losses']) == 0:
            if home_road_split.loc[row['HOME'], 'home_wins'] > 0:
                home_win_per = 1
            else:
                home_win_per = 0
        else:
            home_wins = home_road_split.loc[row['HOME'], 'home_wins']
            home_win_per = home_wins / (home_road_split.loc[row['HOME'], 'home_wins'] + home_road_split.loc[row['HOME'], 'home_losses'])

        if (home_road_split.loc[row['VISITOR'], 'road_wins'] + home_road_split.loc[row['VISITOR'], 'road_losses']) == 0:
            if home_road_split.loc[row['VISITOR'], 'road_wins'] > 0:
                visitor_win_per = 1
            else:
                visitor_win_per = 0
        else:
            visitor_wins = home_road_split.loc[row['VISITOR'], 'road_wins']
            visitor_win_per = visitor_wins / (home_road_split.loc[row['VISITOR'], 'road_wins'] + home_road_split.loc[row['VISITOR'], 'road_losses'])

        home_win_at_home.insert(index, home_win_per)
        visitor_win_on_road.insert(index, visitor_win_per)

        if row['WINNER'] == 1:
            home_road_split.at[row['HOME'],'home_wins'] = home_road_split.loc[row['HOME'],'home_wins'] + 1
            home_road_split.at[row['VISITOR'],'road_losses'] = home_road_split.loc[row['VISITOR'],'road_losses'] + 1
        elif row['WINNER'] == 0:
            home_road_split.at[row['VISITOR'],'road_wins'] = home_road_split.loc[row['VISITOR'],'road_wins'] + 1
            home_road_split.at[row['HOME'],'home_losses'] = home_road_split.loc[row['HOME'],'home_losses'] + 1

    entire_schedule['HOME_W%_AT_HOME'] = home_win_at_home
    entire_schedule['VISITOR_W%_ON_ROAD'] = visitor_win_on_road
    return entire_schedule

def add_second_of_b2b(entire_schedule):
    teams_that_played_last_night = []
    teams_that_played_today = []
    home_team_b2b = []
    visitor_team_b2b = []
    last_night_day = "Monday"

    for index, row in entire_schedule.iterrows():
        day = pd.Timestamp.day_name(row['DATE'])

        if get_next_day(last_night_day) != day:
            teams_that_played_last_night = teams_that_played_today
            teams_that_played_today = []
            last_night_day = get_day_before(day)

        if get_next_day(last_night_day) == day:
            if row['HOME'] in teams_that_played_last_night:
                home_team_b2b.insert(index, 1)
            else:
                home_team_b2b.insert(index, 0)

            if row['VISITOR'] in teams_that_played_last_night:
                visitor_team_b2b.insert(index, 1)
            else:
                visitor_team_b2b.insert(index, 0)

        teams_that_played_today.append(row['HOME'])
        teams_that_played_today.append(row['VISITOR'])

    entire_schedule["HOME_B2B"] = home_team_b2b
    entire_schedule["VISITOR_B2B"] = visitor_team_b2b
    return entire_schedule

def add_last_twenty(entire_schedule):
    team_last_twenty = get_teams_last_twenty_init()
    home_last_twenty_percent = []
    visitor_last_twenty_percent = []
    for index, row in entire_schedule.iterrows():

        home_last_twenty_percent.append(get_win_loss_percentage_last_twenty(row['HOME'], team_last_twenty))
        visitor_last_twenty_percent.append(get_win_loss_percentage_last_twenty(row['VISITOR'], team_last_twenty))

        if not pd.isna(row['WINNER']):
            temp_home = team_last_twenty[row['HOME']]
            temp_home.pop(0)
            temp_visitor = team_last_twenty[row['VISITOR']]
            temp_visitor.pop(0)

            if row['WINNER'] == 1:
                temp_home.append('W')
                temp_visitor.append('L')
                team_last_twenty.update({row['HOME'] : temp_home})
                team_last_twenty.update({row['VISITOR'] : temp_visitor})
            elif row['WINNER'] == 0:
                temp_home.append('L')
                temp_visitor.append('W')
                team_last_twenty.update({row['HOME'] : temp_home})
                team_last_twenty.update({row['VISITOR'] : temp_visitor})

    entire_schedule["HOME_LAST_20"] = home_last_twenty_percent
    entire_schedule["VISITOR_LAST_20"] = visitor_last_twenty_percent
    return entire_schedule

def finalize_csv(name, year):
    entire_schedule = get_schedule(year, playoffs=False)
    entire_schedule = convert_team_names(entire_schedule)
    add_winner_column(entire_schedule)
    entire_schedule = add_win_percentage(add_team_stats(entire_schedule))
    entire_schedule = add_second_of_b2b(entire_schedule)
    entire_schedule = add_home_away_splits(entire_schedule)
    entire_schedule = add_last_twenty(entire_schedule)
    entire_schedule.to_csv(name, index=False)

    sys.stderr.write("[PROGRESS] CSV file creation completed!\n")
    sys.stderr.flush()

    return name


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import os
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'
#设置数据库链接
import cx_Oracle
conn_str = 'tiger/tiger@192.168.1.152:1521/caiji'
connection_oracle = cx_Oracle.Connection(conn_str)
cursor_oracle = connection_oracle.cursor()

# 建立浏览器对象 ，通过Phantomjs
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
browser = webdriver.Chrome(chrome_options=chrome_options)

titles = []

#获取每个公告的内容
def get_each_announcement(link):
	global browser
	browser.get(link)
	browser.implicitly_wait(3)
	disaplay = browser.find_element_by_id('displayGG')
	disaplay.click()
	browser.implicitly_wait(1)

	content = browser.find_element_by_class_name('vF_detail_content')

	return content.text[0:2000]

#获取每个列表的点击URL
def get_visit_links(url):
	print(url)
	global browser
	# 访问url
	browser.get(url)

	# 等待一定时间，让js脚本加载完毕
	browser.implicitly_wait(3)

	# 找到list
	ul = browser.find_element_by_class_name('c_list_bid')
	results = ul.find_elements_by_tag_name('li')

	links = []

	for result in results:
		aa = result.find_element_by_tag_name('a')
		global titles
		titles.append(aa.text)
		# print(aa.text)
		link = aa.get_attribute('href')
		links.append(link)

	return links

if __name__ == "__main__":
    url = 'http://www.ccgp.gov.cn/cggg/dfgg/'
    url_list = []
    url_list.append(url)

    for index in range(1, 24):
    	url_list.append(url + 'index_' + str(index) + ".htm")

    i = 0

    for url in url_list :
    	print(url)
    	links = get_visit_links(url)
    	for link in links :
    		print(link)
    		content = get_each_announcement(link)
    		cursor_oracle.execute("select * from zfgg where url = '" + link + "'")
    		row = cursor_oracle.fetchone()
    		#print(row)
    		#已经存在了，就不加了
    		if row is None:
    			cursor_oracle.callproc('insertgg', ('dfgg', titles[i], content, link))
    		i = i + 1

    connection_oracle.commit()
    cursor_oracle.close()
    connection_oracle.close()


from django.shortcuts import render, HttpResponse, redirect
from account.forms import RegistrationForm, UserProfileForm
from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.contrib import messages
from case.models import case , Task , Document
from case.forms import Reg_Case, Reg_Task , DocumentForm
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
import os
from django.core.files.storage import default_storage

#python imports
import spacy
from spacy import displacy
from collections import Counter
import en_core_web_sm
nlp = spacy.load('en')
import speech_recognition as sr

from bs4 import BeautifulSoup
import requests
import re

def url_to_string(url):
    res = requests.get(url)
    html = res.text
    soup = BeautifulSoup(html, 'html.parser')
    for script in soup(["script", "style", 'aside']):
        script.extract()
    return " ".join(re.split(r'[\n\t]+', soup.get_text()))

def getfreq(temp_data):
    mp_codes={}
    mp_tags={}
    unique_codes=[]
    for i in range(len(temp_data)):
        if temp_data[i][0].lower() in mp_codes.keys():
            mp_codes[temp_data[i][0].lower()]=mp_codes[temp_data[i][0].lower()]+1
        else:
            mp_codes[temp_data[i][0].lower()]=1

        if temp_data[i][0].lower() in mp_tags.keys():
            continue
        else:
            mp_tags[temp_data[i][0].lower()]=temp_data[i][1].lower()

    return mp_codes,mp_tags

"""
ny_bb = url_to_string('https://www.coursera.org/instructor/andrewng')
article = nlp(ny_bb)
len(article.ents)
print(article)
"""

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
#from django.conf.settings import MEDIA_ROOT
#from django.urls import reverse
# Create your views here.

@transaction.atomic
def register(request):
	if request.method == 'POST':
		form = RegistrationForm(request.POST)
		profile_form = UserProfileForm(request.POST)
		if form.is_valid() and profile_form.is_valid():
			user = form.save()
			user.userprofile.designation = profile_form.cleaned_data.get('designation')
			user.userprofile.city = profile_form.cleaned_data.get('city')
			user.userprofile.unq = profile_form.cleaned_data.get('unq')
			user.userprofile.save()
			return redirect('/account/login')
		else:
			messages.error(request,form.errors)
			args = {'form':form, 'profile_form':profile_form}
			return render(request, 'account/reg_form.html', args)
	else:
		form = RegistrationForm()
		profile_form = UserProfileForm()
		args = {'form':form, 'profile_form':profile_form}
		return render(request, 'account/reg_form.html', args)

@login_required
def view_profile(request):
    if request.user.userprofile.designation == 'investigation head':
        new_cases = case.objects.filter(case_info = '',investigation_head = request.user.userprofile)
        ongoing_cases = case.objects.filter(status='Under Investigation',investigation_head=request.user.userprofile).count()
        completed_cases = case.objects.filter(status='Completed',investigation_head=request.user.userprofile).count()
        args = {'user': request.user,'new_cases':new_cases,'ongoing_cases':ongoing_cases,'completed_cases':completed_cases}
    elif request.user.userprofile.designation == 'member':
        form3 = DocumentForm()
        mp_codes={}
        mp_tags={}
        mp_places={}
        probability={}
        main_mp={}
        if request.method == 'POST':
            form3 = DocumentForm()
            print(request.user.userprofile.unq)
            t1 = Task.objects.filter(task_member=request.user.userprofile)
            print(t1)
            k = Document.objects.filter(unq = t1[0].case_number)
            print(t1[0].task_number)
            #1 file
            #print(k[0].file1)
            f = default_storage.open(os.path.join('', str(k[0].file1)), 'r')
            data = f.read()
            f.close()
            #print(data)
            article1 = nlp(data)
            k1 = [(X.text, X.label_) for X in article1.ents]
            print(len(k1))

            #2 file
            f = default_storage.open(os.path.join('', str(k[0].csvfile1)), 'r')
            data = f.read().splitlines()
            f.close()
            for i in range(len(data)):
                t1 = data[i].split(",")
                if t1[0].lower() in mp_places.keys():
                    continue
                else:
                    mp_places[t1[0].lower()]=t1[1].lower()

            print(mp_places)
            #article1 = nlp(data)
            print(len(k1))

            #1 csvfile1
            f = default_storage.open(os.path.join('', str(k[0].audio1)), 'rb')
            #data = f.read()
            #print(f)
            r = sr.Recognizer()
            harvard = sr.AudioFile(f)

            #1 audiofile1
            with harvard as source:
                audio = r.record(source)
            k = r.recognize_google(audio)
            article1 = nlp(k)
            k1.extend([(X.text, X.label_) for X in article1.ents])
            f.close()
            print([(X.text, X.label_) for X in article1.ents])
            print('hello')
            print('hello')
            #k1.extend([(X.text, X.label_) for X in article1.ents])

            print(len(k1))
            print(len(set(k1)))

            mp_codes, mp_tags = getfreq(k1)
            print(mp_codes)
            print(mp_tags)

            total =0
            for i in mp_tags:
                print("i ",mp_codes[i])
                total=total+mp_codes[i]

            for i in mp_codes:
                probability[i] = mp_codes[i]/total

            for i in mp_places:
                places = mp_places[i]
                for j in mp_tags:
                    if(i==j):
                        if places in main_mp.keys():
                            main_mp[places] = main_mp[places]+probability[j]
                        else:
                            main_mp[places] = probability[j]

            print("main ",main_mp)
            print("probabity" , probability)
            print("total ",total)
        form = Reg_Task()
        ongoing_task = Task.objects.filter()
        ongoing_cases = case.objects.filter(status='Under Investigation',members=request.user.userprofile).count()
        completed_cases = case.objects.filter(status='Completed',members=request.user.userprofile).count()
        args = {'mp_main':main_mp , 'mp_codes':mp_codes, 'mp_tags':mp_tags, 'user': request.user, 'ongoing_cases':ongoing_cases,'completed_cases':completed_cases , 'form':form, 'ongoing_task':ongoing_task,'form3':form3}
    else:
        args = {'user':request.user}
        if request.method == 'POST':
            form = Reg_Case(request.POST)
            if form.is_valid():
                form.save()
                return redirect('/account/profile')
            else:
                form = Reg_Case()
            args = {'user':request.user,'form':form}

    return render(request, 'account/profile.html', args)

"""
@login_required
def function1(request):
    form = NewInput1()
    if request.method=="POST":
        form = NewInput1(request.POST)

        if form.is_valid():

			model_instance = form.save(commit=False)
            article1=""
            article2="32"
            article3=""
            article4=""
            article5=""

            try:
                data1 = request.FILES['file1']
                f1 = data1.read().decode('utf-8')#.splitlines()
                article1 = nlp(f1)
                k1 = [(X.text, X.label_) for X in article1.ents]
            except:
                1

            try:
                data2 = request.FILES['file2']
                f2 = data2.read().decode('utf-8')#.splitlines()
                article2 = nlp(f2)
                print(article2)
                k2 = [(X.text, X.label_) for X in article2.ents]
            except:
                1

            try:
                data3 = request.FILES['file3']
                f3 = data3.read().decode('utf-8')#.splitlines()
                article3 = nlp(f3)
                k3 = [(X.text, X.label_) for X in article3.ents]
            except:
                1

            try:
                data4 = request.FILES['csvfile1']
                f4 = data4.read().decode('utf-8')#.splitlines()
                print(f4.split('\n')[3])
            except:
                1

            try:
                ny_bb4 = url_to_string(model_instance.url1)
                article4 = nlp(ny_bb4)
                print(set([(X.text, X.label_) for X in article4.ents]))
            except:
                1

            try:
                ny_bb5 = url_to_string(model_instance.url2)
                articler5 = nlp(ny_bb5)
            except:
                1

            return render(request , 'filetonlp/input1.html', {'form':form , 'article2':article4})

    return render(request , account/profile.html, {'form':form })
"""


import os
import json
import glob
from scipy import sparse
from scipy.sparse import csr_matrix
from nltk.corpus import stopwords
from source.util.file_utils import get_lemmas_list_with_token_tags_from_document_annotation

def get_csr_matrix(docs, tokens_dict, min_fr):
    vocabulary = {}
    indptr = [0]
    indices = []
    data = []
    for d in docs:
        for term in d:
            if tokens_dict[term] >= min_fr:
                index = vocabulary.setdefault(term, len(vocabulary))
                indices.append(index)
                data.append(1)
        indptr.append(len(indices))
    return csr_matrix((data, indices, indptr), dtype=int)

def build_csr_term_doc_matrix(annotation_documents_dir_path, min_fr, dict_file, matrix_file):
    tokens_dict = {}
    eng_stopwords = stopwords.words("english")
    file_counter = 1
    docs = []
    # iterate through annotations
    for subdir, dirs, files in os.walk(annotation_documents_dir_path):
        for file in files:
            current_file_path = str(glob.glob(annotation_documents_dir_path + "/**/" + str(file_counter) + ".tsv", recursive=True)[0])
            doc_tokens_list = get_lemmas_list_with_token_tags_from_document_annotation(current_file_path)
            print(file_counter)
            file_counter += 1
            final_tokens_list_for_document = []
            # iterate though tokens from annotation
            for token_info in doc_tokens_list:
                # check for stop words and punctuation
                if (token_info[0].lower() not in eng_stopwords) and (token_info[1] != "punctuation sign") \
                        and (token_info[1] != "quotation") and (token_info[1] != "word with unicode decimal code")\
                        and (token_info[1] != "undefined token type"):
                    token = str(token_info[0].lower())
                    # check if token exist in dict and create or inc value
                    if not (token in tokens_dict):
                        tokens_dict[token] = 0
                    else:
                        tokens_dict[token] += 1
                    # add to final list
                    final_tokens_list_for_document.append(token)
            docs.append(final_tokens_list_for_document)

    # filter resulting dict by frequency
    tokens_dict_to_save = {k: v for k, v in tokens_dict.items() if tokens_dict[k] >= min_fr}
    # building and writing to file dictionary
    f1 = open(dict_file, "w")
    json.dump(tokens_dict_to_save, f1)
    f1.close()
    # getting csr matrix
    td_matrix = get_csr_matrix(docs, tokens_dict, min_fr)
    sparse.save_npz(matrix_file, td_matrix)


from PIL import Image
from torchvision import transforms, models
import torchvision
import torch
import pickle
import timeit

start = timeit.default_timer()
vgg2 = models.vgg16(pretrained=True)
device = torch.device("cuba" if torch.cuda.is_available() else "cpu")
vgg2.to(device)
n = 1512
vectors = []
for i in range(n):
    # load the images from folder one by one (Change to the desired input name)
    img_path = f'../database/image_224X224/{i}.jpeg'
    img = Image.open(img_path).convert('RGB')
    in_transform = transforms.Compose([
        torchvision.transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406],
                             std=[0.229, 0.224, 0.225])])
    img_tensor = in_transform(img).unsqueeze(0)
    input_tensor = img_tensor.to(device)
    input_vgg = vgg2(input_tensor).detach().to('cpu')
    # print(input_vgg[0].tolist())
    vectors.append(input_vgg[0].tolist())

# Save the vectors to binary file (Change to the desired output name)
with open('../database/image_newDataBase_224X224.pkl', 'wb') as f:
    pickle.dump(vectors, f)

stop = timeit.default_timer()
print('Time: ', stop - start)


import logging
import torch
import torch.optim as optim

from nni.nas.pytorch.trainer import Trainer
from nni.nas.pytorch.utils import AverageMeterGroup
from .mutator import EnasMutator

logger = logging.getLogger(__name__)

class EnasTrainer(Trainer):
    def __init__(self, model, loss, metrics, reward_function,
                 optimizer, num_epochs, dataset_train, dataset_valid,
                 mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None,
                 entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999,
                 mutator_lr=0.00035, mutator_steps_aggregate=20, mutator_steps=50, aux_weight=0.4):
        super().__init__(model, mutator if mutator is not None else EnasMutator(model),
                         loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid,
                         batch_size, workers, device, log_frequency, callbacks)
        self.reward_function = reward_function
        self.mutator_optim = optim.Adam(self.mutator.parameters(), lr=mutator_lr)

        self.entropy_weight = entropy_weight
        self.skip_weight = skip_weight
        self.baseline_decay = baseline_decay
        self.baseline = 0.
        self.mutator_steps_aggregate = mutator_steps_aggregate
        self.mutator_steps = mutator_steps
        self.aux_weight = aux_weight

        n_train = len(self.dataset_train)
        split = n_train // 10
        indices = list(range(n_train))
        train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split])
        valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:])
        self.train_loader = torch.utils.data.DataLoader(self.dataset_train,
                                                        batch_size=batch_size,
                                                        sampler=train_sampler,
                                                        num_workers=workers)
        self.valid_loader = torch.utils.data.DataLoader(self.dataset_train,
                                                        batch_size=batch_size,
                                                        sampler=valid_sampler,
                                                        num_workers=workers)
        self.test_loader = torch.utils.data.DataLoader(self.dataset_valid,
                                                       batch_size=batch_size,
                                                       num_workers=workers)

    def train_one_epoch(self, epoch):
        # Sample model and train
        self.model.train()
        self.mutator.eval()
        meters = AverageMeterGroup()
        for step, (x, y) in enumerate(self.train_loader):
            x, y = x.to(self.device), y.to(self.device)
            self.optimizer.zero_grad()

            with torch.no_grad():
                self.mutator.reset()
            logits = self.model(x)

            if isinstance(logits, tuple):
                logits, aux_logits = logits
                aux_loss = self.loss(aux_logits, y)
            else:
                aux_loss = 0.
            metrics = self.metrics(logits, y)
            loss = self.loss(logits, y)
            loss = loss + self.aux_weight * aux_loss
            loss.backward()
            self.optimizer.step()
            metrics["loss"] = loss.item()
            meters.update(metrics)

            if self.log_frequency is not None and step % self.log_frequency == 0:
                logger.info("Model Epoch [%s/%s] Step [%s/%s]  %s", epoch + 1,
                            self.num_epochs, step + 1, len(self.train_loader), meters)

        # Train sampler (mutator)
        self.model.eval()
        self.mutator.train()
        meters = AverageMeterGroup()
        mutator_step, total_mutator_steps = 0, self.mutator_steps * self.mutator_steps_aggregate
        while mutator_step < total_mutator_steps:
            for step, (x, y) in enumerate(self.valid_loader):
                x, y = x.to(self.device), y.to(self.device)

                self.mutator.reset()
                with torch.no_grad():
                    logits = self.model(x)
                metrics = self.metrics(logits, y)
                reward = self.reward_function(logits, y)
                if self.entropy_weight is not None:
                    reward += self.entropy_weight * self.mutator.sample_entropy
                self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay)
                self.baseline = self.baseline.detach().item()
                loss = self.mutator.sample_log_prob * (reward - self.baseline)
                if self.skip_weight:
                    loss += self.skip_weight * self.mutator.sample_skip_penalty
                metrics["reward"] = reward
                metrics["loss"] = loss.item()
                metrics["ent"] = self.mutator.sample_entropy.item()
                metrics["baseline"] = self.baseline
                metrics["skip"] = self.mutator.sample_skip_penalty

                loss = loss / self.mutator_steps_aggregate
                loss.backward()
                meters.update(metrics)

                if mutator_step % self.mutator_steps_aggregate == 0:
                    self.mutator_optim.step()
                    self.mutator_optim.zero_grad()

                if self.log_frequency is not None and step % self.log_frequency == 0:
                    logger.info("RL Epoch [%s/%s] Step [%s/%s]  %s", epoch + 1, self.num_epochs,
                                mutator_step // self.mutator_steps_aggregate + 1, self.mutator_steps, meters)
                mutator_step += 1
                if mutator_step >= total_mutator_steps:
                    break

    def validate_one_epoch(self, epoch):
        pass


import os

nimage = os.listdir('testCifar/apple/')
count1 = 0
for j in nimage:
    count1+=1
print(count1)
image = os.listdir('Cifar/test/apple')
count2 = 0
for j in image:
    count2+=1
print(count2)
count=count1/count2
print('新数据集图像数是原数据集的{}倍。'.format(count))


# this program prints the fibonacci series for the number of inputs the user enters
# asking user to input the number of terms needed
num = int(input("Enter the number of terms needed: "))
# initializing variables to display the fibonacci series
n1 = 1
n2 = 1
val = 0
# creating a loop to display fibonacci series
for i in range(num):
    print(n2)
    val = n1
    n1 = n1 + n2
    n2 = val


import asyncio

import semantic_kernel as sk
import semantic_kernel.ai.open_ai as sk_oai

import glob
import os
from typing import Dict

from semantic_kernel.kernel_extensions.extends_kernel import ExtendsKernel
from semantic_kernel.orchestration.sk_function_base import SKFunctionBase
from semantic_kernel.semantic_functions.prompt_template import PromptTemplate
from semantic_kernel.semantic_functions.prompt_template_config import (
    PromptTemplateConfig,
)
from semantic_kernel.semantic_functions.semantic_function_config import (
    SemanticFunctionConfig,
)
from semantic_kernel.utils.validation import validate_skill_name

system_message = """
"""

kernel = sk.Kernel()

from semantic_kernel.ai.open_ai import AzureChatCompletion
from semantic_kernel.ai.open_ai import AzureTextCompletion

deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()

kernel.config.add_chat_backend("dv", AzureChatCompletion(deployment, endpoint, api_key))
kernel.config.add_text_backend("text", AzureTextCompletion(deployment, endpoint, api_key))

prompt_config = sk.PromptTemplateConfig.from_completion_parameters(
    max_tokens=2000, temperature=0.7, top_p=0.8
)

async def chat() -> bool:
    context_vars = sk.ContextVariables()

    try:
        user_input = input("User:> ")
        context_vars["user_input"] = user_input
    except KeyboardInterrupt:
        print("\n\nExiting chat...")
        return False
    except EOFError:
        print("\n\nExiting chat...")
        return False

    if user_input == "exit":
        print("\n\nExiting chat...")
        return False

    assist = ""

    skills = import_chat_skill_from_directory("./questions", "PatternMatchingSkill")
    # loop skill and call skill
    for key, skill in skills.items():
        context = skill(user_input)
        print(f"{key}:> {context}")

    print(f"assist:> {assist}")
    return True

async def main() -> None:
    chatting = True
    while chatting:
        chatting = await chat()

def import_chat_skill_from_directory(
    parent_directory: str, skill_directory_name: str
    ) -> Dict[str, SKFunctionBase]:
        CONFIG_FILE = "config.json"
        PROMPT_FILE = "skprompt.txt"

        kernel =  sk.Kernel()

        validate_skill_name(skill_directory_name)

        skill_directory = os.path.join(parent_directory, skill_directory_name)
        skill_directory = os.path.abspath(skill_directory)

        if not os.path.exists(skill_directory):
            raise ValueError(f"Skill directory does not exist: {skill_directory_name}")

        skill = {}

        directories = glob.glob(skill_directory + "/*/")
        for directory in directories:
            dir_name = os.path.dirname(directory)
            function_name = os.path.basename(dir_name)
            prompt_path = os.path.join(directory, PROMPT_FILE)

            # Continue only if the prompt template exists
            if not os.path.exists(prompt_path):
                continue

            config = PromptTemplateConfig()
            config_path = os.path.join(directory, CONFIG_FILE)
            with open(config_path, "r") as config_file:
                config = config.from_json(config_file.read())

            # Load Prompt Template
            with open(prompt_path, "r") as prompt_file:
                template = sk.ChatPromptTemplate(
                    prompt_file.read(), kernel.prompt_template_engine, config
                )

            # Prepare lambda wrapping AI logic
            function_config = SemanticFunctionConfig(config, template)

            skill[function_name] = kernel.register_semantic_function(
                skill_directory_name, function_name, function_config
            )

if __name__ == "__main__":
    asyncio.run(main())


import json
import os
import platform

import svtplay_dl.subtitle
from svtplay_dl.utils.parser import setup_defaults

def parse(subfile):
    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "subtitle", subfile), encoding="utf8") as fd:
        data = fd.read()
    return data

def parse2(subfile):
    encoding = None
    if platform.system() == "Windows":
        encoding = "utf8"
    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "subtitle", subfile), encoding=encoding) as fd:
        data = fd.read()
    return data

def test_wsrt_old():
    data = parse("wsrt-old.srt")
    subtitle = svtplay_dl.subtitle.subtitle(setup_defaults(), "", "")
    assert subtitle._wrst(data) == "1\n01:23:45,678 --> 01:23:46,789\nHello world!\n\n2\n01:23:48,910 --> 01:23:49,101\nHello\nworld!\n"

def test_wsrt_pretext():
    data = parse("wsrt-pretext.srt")
    subtitle = svtplay_dl.subtitle.subtitle(setup_defaults(), "", "")
    assert (
        subtitle._wrst(data)
        == "1\n00:00:10,040 --> 00:00:12,520\nNån försökte slå ihjäl mig.\n\n2\n00:00:12,600 --> 00:00:16,200\n-Var fan har du varit?\n-Vad?\n"
    )

def test_wsrt_style_hash():
    data = parse("wsrt-style-hash.srt")
    subtitle = svtplay_dl.subtitle.subtitle(setup_defaults(), "", "")
    assert (
        subtitle._wrst(data)
        == "1\n00:00:06,360 --> 00:00:10,080\n1845 gav sig\nen brittisk expedition ut-\n\n2\n00:00:10,240 --> 00:00:15,040\n-för att söka ett av forsknings-\nresornas mest eftertraktade mål:\n"
    )

def test_wrst_segment():
    dataj = json.loads(parse("wsrt-segments.json"))
    data = parse2("wrst-correct.srt") + "\n"  # last \n to fix UT with extra new line
    assert svtplay_dl.subtitle._wrstsegments(dataj) == data

def test_wrst_segment2():
    dataj = json.loads(parse("wsrt-segments-2.json"))
    data = parse2("wrst-correct-2.srt") + "\n"  # last \n to fix UT with extra new line
    assert svtplay_dl.subtitle._wrstsegments(dataj) == data


from django.urls import path
from . import views

app_name = 'qblog'

urlpatterns = [
    path('', views.PostList.as_view(), name='blog'),
    path('create/', views.PostCreateView.as_view(), name='post_create'),
    path('<slug:slug>/', views.PostDetail.as_view(), name='post_detail'),
    path('edit/<slug:slug>/', views.PostEditView.as_view(), name='edit_post'),
    path('like/<slug:slug>', views.PostLike.as_view(), name='post_like'),
]


import pygame
import math
import random
class Cursor:

    def __init__(self, screen, width):
        self.start = pygame.mouse.get_pos()
        self.colour = [250, 250, 250]
        self.screen = screen
        self.width = width
        self.cursor = pygame.image.load('cursor.png')
        self.cursor = pygame.transform.scale(self.cursor, (20, 20))

    def update(self):

        curr_pos = [i-10 for i in list(pygame.mouse.get_pos())]

        self.screen.blit(self.cursor, curr_pos)

class Planet:

    def __init__(self, screen, id, GRAV):
        self.screen = screen
        self.last_pos = None
        self.pos = list(pygame.mouse.get_pos())
        self.GRAV = 1
        self.DENSITY = 1
        self.radius = 1
        self.growing = True
        self.id = id
        self.vel = [0,0] if self.id == 0 else  [random.uniform(0,0.10),random.uniform(0,0.10)]

    def grow(self):
        if pygame.mouse.get_pressed()[0] == 1 and self.growing:
            self.radius += 0.5
        else:
            self.growing = False
            print(self.radius)

    def get_volume(self):
        return (4/3) * math.pi * ((self.radius) ** 3)

    def get_mass(self):
        return self.get_volume() * self.DENSITY

    def set_vels(self, planets):
        if not self.growing:
            for planet in planets:
                if planet.id != self.id and not planet.growing:
                    dt = 0.00001
                    #r_hat = self.pos
                    # calculate x and y total distance
                    r = tuple(map(lambda i, j: i -j, planet.pos, self.pos))
                    distance = math.sqrt(r[0]**2 + r[1]**2)
                    # calculate gravitational force
                    G = self.GRAV * ((planet.get_mass() * self.get_mass()) / (distance ** 2))

                    # get rs
                    r_mag = math.sqrt(r[0]**2 + r[1]**2)
                    r_hat = tuple([i/r_mag for i in r])

                    r_hat_x = ((self.GRAV * (self.get_mass()/r_mag**2)) * r_hat[0]) * dt
                    r_hat_y = ((self.GRAV * (self.get_mass()/r_mag**2)) * r_hat[1]) * dt
                    planet.vel[0] -= r_hat_x
                    planet.vel[1] -= r_hat_y

    def update_pos(self):
        self.pos[0] += self.vel[0]
        self.pos[1] += self.vel[1]
        self.last_pos = self.pos

    def update(self, planets):

        if self.growing:
            self.grow()

        self.set_vels(planets)
        self.update_pos()
        #print(self.vel, self.pos)
        pygame.draw.circle(self.screen, (255,255,255), self.pos, self.radius)
        if self.id == 1:
            print(self.pos)


import turtle
import random

# 전역 변수 선언 #
myTurtle, tX, tY, tColor, tSize, tShape = [None] * 6
shapeList = []
playerTurtles = [] #거북이 2차원 리스트
swidth, sheight = 500, 500

# 메인 코드 #
if __name__ == "__main__" :  #이건 왜 선언을 하는거지? 인터프리터에서 직접 실행한 경우에만 if 문 코드 실행
    turtle.title('거북 리스트 활용')
    turtle.setup(width = swidth + 50, height = sheight + 50)
    turtle.screensize(swidth,sheight)

    shapeList = turtle.getshapes() # 모든 거북이 이름 목록 반환
    # getshapes() = ['arrow','blank','circle','classic','square','triangle','turtle]
    for i in range(0,5) : #tutle object 100개
        random.shuffle(shapeList)
        myTurtle = turtle.Turtle(shapeList[0])  # 맨 처음 화살표로 출발하기 위함
        tX = random.randrange(-swidth/2 , swidth/2)
        tY = random.randrange(-sheight/2 , sheight/2)
        r = random.random()
        g = random.random()
        b = random.random()

        tSize = random.randrange(1,3)       # 나오는 모양의 크기가 1~3이 되게
        playerTurtles.append([myTurtle, tX, tY, tSize, r, g, b])

    for tList in playerTurtles : # playerTurtles 안에 tList만큼 반복?
        myTurtle = tList[0]
        myTurtle.color ((tList[4], tList[5], tList[6]))
        myTurtle.pencolor((tList[4],tList[5],tList[6]))
        myTurtle.turtlesize(tList[3])
        myTurtle.penup()
        myTurtle.setpos(tList[1], tList[2])
        myTurtle.pendown()
        myTurtle.goto(0,0)

    turtle.done()


"""The FastTextAreaInput extends the Panel TextAreaInput to a Fast Design Framework TextAreaInput.

It is built on the the fast-text-area web component. The component supports two visual apperances
(outline and filled).

For more information view the [component specification]\
(https://github.com/microsoft/fast/tree/main/packages/web-components/fast-foundation/src/\
text-area).

See also https://explore.fast.design/components/fast-text-area.
    """
import panel as pn
import param  # pylint: disable=wrong-import-order
from panel.widgets import TextAreaInput

from awesome_panel_extensions.bokeh_extensions.fast.fast_textarea_input import (
    FastTextAreaInput as _BkFastTextAreaInput,
)

FAST_TEXT_AREA_APPEARENCES = [
    "outline",
    "filled",
]
DEFAULT_TEXT_AREA_APPEARANCE = None
RESIZES = [None, "both", "horizontal", "vertical"]
DEFAULT_RESIZE = None

class _FastTextAreaInputMixin(pn.widgets.Widget):
    # value  is inherited
    # placeholder is inherited
    # disabled is inherited
    # list is not supported
    placeholder = param.String(
        default="", doc="A placeholder string displayed when no value is entered."
    )
    appearance = param.ObjectSelector(
        default=DEFAULT_TEXT_AREA_APPEARANCE,
        objects=FAST_TEXT_AREA_APPEARENCES,
        doc="""Determines the appearance of the TextAreaInput. One of None, `outline` or `filled`.
        Defaults to None which is the same a `outline`""",
        allow_None=True,
    )
    autofocus = param.Boolean(
        default=False,
        doc="""The autofocus attribute. Defaults to `False`""",
    )
    resize = param.ObjectSelector(
        default=DEFAULT_RESIZE,
        objects=RESIZES,
        constant=True,
        doc="""The resize attribute. One of
        `None`, `both`, `horizontal` or `vertical`. Defaults to `None`.
        Currently constant since Panel does not (yet?) support resizing widgets by dragging.
        """,
        allow_None=True,
    )
    cols = param.Integer(default=20, bounds=(0, None))
    rows = param.Integer(
        doc="The number of visible text lines for the control.", default=2, bounds=(0, None)
    )
    min_length = param.Integer(
        default=0,
        doc="""The minimum length of the text string""",
    )
    spellcheck = param.Boolean(
        default=False, doc="""Whether or not the spell check is enabled. Default is False"""
    )
    required = param.Boolean(
        default=False, doc="""Whether or not the FastTextAreaInput is required. Default is False"""
    )
    readonly = param.Boolean(
        default=False, doc="""Whether or not the FastTextAreaInput is readonly. Default is False"""
    )

    # If we don't set this then the pane below will overlap
    height = param.Integer(default=100, bounds=(0, None))

    _widget_type = _BkFastTextAreaInput

class FastTextAreaInput(_FastTextAreaInputMixin, TextAreaInput):
    """The FastTextAreaInput extends the Panel TextAreaInput to a Fast Design Framework
TextAreaInput.

It is built on the the fast-text-area web component. The component supports two visual apperances
(outline and filled).

For more information view the [component specification]\
(https://github.com/microsoft/fast/tree/main/packages/web-components/fast-foundation/src/\
text-area).

See also https://explore.fast.design/components/fast-text-area.
    """

    _rename = {
        **pn.widgets.TextAreaInput._rename,  # pylint: disable=protected-access
    }


"""Radiance Glow Material.

http://radsite.lbl.gov/radiance/refer/ray.html#Glow
"""
from __future__ import division

from .materialbase import Material
import honeybee.typing as typing

class Glow(Material):
    """Create glow material.

    Args:
        identifier: Text string for a unique Material ID. Must not contain spaces
            or special characters. This will be used to identify the object across
            a model and in the exported Radiance files.
        r_emittance: A positive value for the Red channel of the glow (default: 0).
        g_emittance: A positive value for the Green channel of the glow (default: 0).
        b_emittance: A positive value for the Blue channel of the glow (default: 0).
        max_radius: Maximum radius for shadow testing (default: 0). If maxrad is
            zero, then the surface will never be tested for shadow, although it may
            participate in an interreflection calculation. If maxrad is negative,
            then the surface will never contribute to scene illumination. Glow
            sources will never illuminate objects on the other side of an illum
            surface. This provides a convenient way to illuminate local light fixture
            geometry without overlighting nearby objects.
        modifier: Material modifier (Default: None).
        dependencies: A list of primitives that this primitive depends on. This
            argument is only useful for defining advanced primitives where the
            primitive is defined based on other primitives. (Default: [])

    Properties:
        * identifier
        * display_name
        * r_emittance
        * g_emittance
        * b_emittance
        * max_radius
        * values
        * modifier
        * dependencies
        * is_modifier
        * is_material
    """

    __slots__ = ('_r_emittance', '_g_emittance', '_b_emittance', '_max_radius')

    def __init__(self, identifier, r_emittance=0.0, g_emittance=0.0, b_emittance=0.0,
                 max_radius=0.0, modifier=None, dependencies=None):
        """Init Glow material."""
        Material.__init__(self, identifier, modifier=modifier,
                          dependencies=dependencies)
        self.r_emittance = r_emittance
        self.g_emittance = g_emittance
        self.b_emittance = b_emittance
        self.max_radius = max_radius
        self._update_values()

    def _update_values(self):
        "update value dictionaries."
        self._values[2] = [
            self.r_emittance, self.g_emittance, self.b_emittance, self.max_radius
        ]

    @property
    def r_emittance(self):
        """Get or set a positive value for the Red channel of the glow.

        The value must be positive (Default: 0).
        """
        return self._r_emittance

    @r_emittance.setter
    def r_emittance(self, value):
        self._r_emittance = typing.float_positive(value)

    @property
    def g_emittance(self):
        """Get or set a positive value for the Green channel of the glow.

        The value must be positive (Default: 0).
        """
        return self._g_emittance

    @g_emittance.setter
    def g_emittance(self, value):
        self._g_emittance = typing.float_positive(value)

    @property
    def b_emittance(self):
        """Get or set a positive value for the Blue channel of the glow.

        The value must be positive (Default: 0).
        """
        return self._b_emittance

    @b_emittance.setter
    def b_emittance(self, value):
        self._b_emittance = typing.float_positive(value)

    @property
    def max_radius(self):
        """Get or set the maximum radius for shadow testing. (Default: 0).

        If maxrad is zero, then the surface will never be tested for shadow, although
        it may participate in an interreflection calculation. If maxrad is negative, then
        the surface will never contribute to scene illumination. Glow sources will
        never illuminate objects on the other side of an illum surface. This
        provides a convenient way to illuminate local light fixture geometry without
        overlighting nearby objects.
        """
        return self._max_radius

    @max_radius.setter
    def max_radius(self, value):
        self._max_radius = float(value)

    @classmethod
    def from_single_value(cls, identifier, rgb=0, max_radius=0, modifier=None,
                          dependencies=None):
        """Create glow material with single value.

        Args:
            identifier: Text string for a unique Material ID. Must not contain spaces
            or special characters. This will be used to identify the object across
            a model and in the exported Radiance files.
            rgb: Input for r_emittance, g_emittance and b_emittance. The value should be
                between 0 and 1 (Default: 0).
            modifier: Material modifier (Default: None).
            max_radius: Maximum radius for shadow testing (default: 0). If maxrad is zero,
                then the surface will never be tested for shadow, although it may
                participate in an interreflection calculation. If maxrad is negative, then
                the surface will never contribute to scene illumination. Glow sources will
                never illuminate objects on the other side of an illum surface. This
                provides a convenient way to illuminate local light fixture geometry without
                overlighting nearby objects.
            dependencies: A list of primitives that this primitive depends on. This
                argument is only useful for defining advanced primitives where the
                primitive is defined based on other primitives. (Default: [])

        Usage:

        .. code-block:: python

            sample_glow = Glow.from_single_value("sample_glow", 100)
            print(sample_glow)
        """
        return cls(identifier, r_emittance=rgb, g_emittance=rgb, b_emittance=rgb,
            max_radius=max_radius, modifier=modifier, dependencies=dependencies)

    @classmethod
    def from_primitive_dict(cls, primitive_dict):
        """Initialize Glow from a primitive dict.

        Args:
            data: A dictionary in the format below.

        .. code-block:: python

            {
            "modifier": {}  # primitive modifier (Default: None)
            "type": "glow",  # primitive type
            "identifier": "",  # primitive identifier
            "display_name": "",  # primitive display name
            "values": [],  # values
            "dependencies": []
            }
        """
        cls._dict_type_check(cls.__name__, primitive_dict)
        modifier, dependencies = cls.filter_dict_input(primitive_dict)
        values = primitive_dict['values'][2]

        cls_ = cls(
            identifier=primitive_dict["identifier"],
            r_emittance=values[0],
            g_emittance=values[1],
            b_emittance=values[2],
            max_radius=values[3],
            modifier=modifier,
            dependencies=dependencies
        )
        if 'display_name' in primitive_dict and primitive_dict['display_name'] is not None:
            cls_.display_name = primitive_dict['display_name']

        # this might look redundant but it is NOT. see glass for explanation.
        cls_.values = primitive_dict['values']
        return cls_

    @classmethod
    def from_dict(cls, data):
        """Initialize Glow from a dictionary.

        Args:
            data: A dictionary in the format below.

        .. code-block:: python

            {
            "type": "Glow",  # primitive type
            "identifier": "",  # Material identifier
            "display_name": "",  # Material display name
            "r_emittance": float,  # A positive value for the Red channel of the glow
            "g_emittance": float,  # A positive value for the Green channel of the glow
            "b_emittance": float,  # A positive value for the Blue channel of the glow
            "max_radius": float,  # Maximum radius for shadow testing
            "modifier": {}  # primitive modifier (Default: None)
            "dependencies: []
            }
        """
        cls._dict_type_check(cls.__name__, data)
        modifier, dependencies = cls.filter_dict_input(data)

        new_obj = cls(identifier=data['identifier'],
                      r_emittance=data['r_emittance'],
                      g_emittance=data['g_emittance'],
                      b_emittance=data['b_emittance'],
                      max_radius=data['max_radius'],
                      modifier=modifier,
                      dependencies=dependencies)
        if 'display_name' in data and data['display_name'] is not None:
            new_obj.display_name = data['display_name']
        return new_obj

    def to_dict(self):
        """Translate this object to a dictionary."""
        base = {
            'modifier': self.modifier.to_dict(),
            'type': 'Glow',
            'identifier': self.identifier,
            'r_emittance': self.r_emittance,
            'g_emittance': self.g_emittance,
            'b_emittance': self.b_emittance,
            'max_radius': self.max_radius,
            'dependencies': [dp.to_dict() for dp in self.dependencies]
        }
        if self._display_name is not None:
            base['display_name'] = self.display_name
        return base

    def __copy__(self):
        mod, depend = self._dup_mod_and_depend()
        new_obj = self.__class__(self.identifier, self.r_emittance, self.g_emittance,
                                 self.b_emittance, self.max_radius, mod, depend)
        new_obj._display_name = self._display_name
        return new_obj


from typing import List
from collections import defaultdict
from collections import deque
import heapq
class Twitter:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.dictfee = defaultdict(set)
        self.dicttweet = defaultdict(deque)
        self.timer = 0

    def postTweet(self, userId: int, tweetId: int) -> None:
        """
        Compose a new tweet.
        """

        self.dicttweet[userId].append((self.timer,tweetId))
        self.timer+=1
        if len(self.dicttweet[userId])>10:
            self.dicttweet[userId].popleft()

    def getNewsFeed(self, userId: int) -> List[int]:
        """
        Retrieve the 10 most recent tweet ids in the user's news feed. Each item in the news feed must be posted by users who the user followed or by the user herself. Tweets must be ordered from most recent to least recent.
        """
        feed = list(self.dicttweet[userId])
        heapq.heapify(feed)
        for followee in self.dictfee[userId]:
            for time,tweet in self.dicttweet[followee]:
                if len(feed)< 10:
                    heapq.heappush(feed,(time,tweet))
                elif time > feed[0][0]:
                    heapq.heappushpop(feed,(time,tweet))
        ans = []
        while len(feed)>0:
            ans.append(heapq.heappop(feed)[1])
        return ans[::-1]

    def follow(self, followerId: int, followeeId: int) -> None:
        """
        Follower follows a followee. If the operation is invalid, it should be a no-op.
        """
        if followerId != followeeId:
            self.dictfee[followerId].add(followeeId)

    def unfollow(self, followerId: int, followeeId: int) -> None:
        """
        Follower unfollows a followee. If the operation is invalid, it should be a no-op.
        """
        if followeeId in self.dictfee[followerId]:
            self.dictfee[followerId].remove(followeeId)

# Your Twitter object will be instantiated and called as such:
# obj = Twitter()
# obj.postTweet(userId,tweetId)
# param_2 = obj.getNewsFeed(userId)
# obj.follow(followerId,followeeId)
# obj.unfollow(followerId,followeeId)
T= Twitter()
T.postTweet(1,5)
print(T.getNewsFeed(1))
T.follow(1,2)
T.postTweet(2,6)
print(T.getNewsFeed(1))
T.unfollow(1,2)
print(T.getNewsFeed(1))


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.

author: Sammi Chekroud
"""
import numpy   as np
import scipy   as sp
from scipy import interpolate
import copy
import sys
import os

class DataError(Exception):
    '''
    Raised when there is an error in the datafile and expectations do not align with what is present
    '''
    def _init_(self, msg):
        self.msg = msg

class ArgumentError(Exception):
    """
    Raised when there is an error in the arguments used in a function here
    """
    def _init_(self, msg):
        self.msg = msg

def Eucdist(x1, y1, x2, y2):
    """
    calculate euclidian distance between two points

    formula: sqrt( (x2-x1)^2 + (y2-y1)^2 )

    """
    distance = np.sqrt( (x2-x1)**2 + (y2-y1)**2)
    return distance

def parse_eye_data(eye_fname, block_rec, trial_rec, nblocks, ntrials = None):
    """

    eye_fname    -- full path to a file to be parsed. this is used as a template to create the parsed data in pickle form

    block_rec    -- boolean as to whether you stopped/started recording blockwise in your task

    trial_rec    -- boolean as to whethre you stopped/started recording trialwise in your task

    nblocks      -- the number of blocks of task run in your experiment. This must be defined in all cases.

    ntrials      -- the number of trials in your data if you recorded the eyes with trialwise stop/start
                    if you recorded blockwise, then ntrials is not needed.

                    if you recorded trialwise, then the number of trials per block
                    is calculated and used to separate the data into defined blocks
    """
    if not os.path.exists(eye_fname): #check that the path to the file exists
        raise Exception('the filename: %s does not exist. Please check!' %eye_fname)

    if 'block_rec' not in locals() and 'trial_rec' not in locals():
        raise ArgumentError('Please indicate whether the recording was stopped/started on each trial or each block')

    if 'block_rec' in locals():
        if block_rec == True:
            trial_rec = False
    if 'trial_rec' in locals():
        if trial_rec == True:
            block_rec = False

    if 'block_rec' in locals():
        if block_rec == True and 'nblocks' not in locals():
            raise ArgumentError('Please specify how many blocks you expect in your data recording')

    if 'trial_rec' in locals():
        if trial_rec == True and 'ntrials' not in locals():
            raise ArgumentError('Please specify how many trials you expect in your data recording')
        elif trial_rec == True and 'ntrials' in locals() and ntrials == None:
            raise ArgumentError('Please specify a number of trials to expect')
        elif trial_rec == True and 'ntrials' in locals() and ntrials != None and 'nblocks' not in locals():
            raise ArgumentError('Please specify how many blocks of data you recorded')

    if block_rec:
        d = _parse_eye_data_blockwise(eye_fname, nblocks)
    else:
        d = _parse_eye_data_trialwise(eye_fname, nblocks, ntrials)

    return d #return the parsed data

def _parse_eye_data_blockwise(eye_fname, nblocks):
    """
    this function is called by parse_eye_data and will operate on data where
    the recording was stopped/started for each block of task data
    """

    d = open(eye_fname, 'r')
    raw_d = d.readlines()
    d.close()

    split_d = []
    for i in range(len(raw_d)):
        tmp = raw_d[i].split()
        split_d.append(tmp)

    start_inds = [x for x in range(len(split_d)) if len(split_d[x]) == 6 and split_d[x][0] == 'START']
    if len(start_inds) != nblocks:
        raise DataError('%d blocks are found in the data, not %d as has been input in nblocks' %(len(start_inds),nblocks))

    #get points where recording stopped
    end_inds   = [x for x in range(len(split_d)) if len(split_d[x]) == 7 and split_d[x][0] == 'END']

    if len(start_inds) != len(end_inds):
        raise DataError('the number of times the recording was started and stopped does not align. check problems with acquisition')

    #assign some empty lists to get filled with information
    trackertime = []
    lx   = []; rx   = []
    ly   = []; ry   = []
    lp   = []; rp   = []

    Efix = []; Sfix = []
    Esac = []; Ssac = []
    Eblk = []; Sblk = []
    Msg  = []

    for i in range(len(start_inds)):
        start_line = start_inds[i]
        end_line   = end_inds[i]

        iblk = np.array(split_d[start_line:end_line]) #get only lines with info for this block

        iblk_event_inds, iblk_events     = [], []
        iblk_blink_inds, iblk_blink      = [], []
        iblk_fix_inds, iblk_fix          = [], []
        iblk_sac_inds, iblk_sac          = [], []
        iblk_input_inds                  = []
        for x,y in np.ndenumerate(iblk):
            if y[0] == 'MSG':
                iblk_event_inds.append(x[0]) # add line index where a message was sent to the eyetracker
                iblk_events.append(y)        # add the line itself to list
            elif y[0] in ['EFIX', 'SFIX']:
                iblk_fix_inds.append(x[0])   # add line index where fixation detected (SR research)
                iblk_fix.append(y)           # add fixation event structure to list
            elif y[0] in ['ESACC', 'SSACC']:
                iblk_sac_inds.append(x[0])   # add line index where saccade detected (SR research)
                iblk_sac.append(y)           # add saccade event structure to list
            elif y[0] in ['EBLINK', 'SBLINK']:
                iblk_blink_inds.append(x[0]) # add line index where blink detected (SR research)
                iblk_blink.append(y)         # add blink event structure to list
            elif y[0] == 'INPUT':
               iblk_input_inds.append(x[0])  # find where 'INPUT' is in data (sometimes appears, has no use...)

        #get all non-data line indices
        iblk_nondata    = sorted(iblk_blink_inds + iblk_sac_inds + iblk_fix_inds + iblk_event_inds + iblk_input_inds)

        iblk_data = np.delete(iblk, iblk_nondata) #remove these lines so all you have is the raw data

        iblk_data = iblk_data[6:] #remove first five lines as these are filler after the recording starts

        try:
            iblk_data = np.vstack(iblk_data)
        except ValueError:
            for item in range(len(iblk_data)):
                iblk_data[item] = iblk_data[item][:7] #make sure each item has only 7 strings. sometimes the eyetracker hasn't appended the random .... at the end
            iblk_data = np.vstack(iblk_data) #shape of this should be number of columns in the file (data)

        iblk_data = iblk_data[:,:7]            # only take first 7 columns as these contain data of interest. last one is redundant (if exists)

        #before you can convert to float, need to replace missing data where its '.' as nans (in pupil this is '0.0')
        eyecols = [1,2,4,5] #leftx, lefty, rightx, right y col indices
        for col in eyecols:
            missing_inds = np.where(iblk_data[:,col] == '.') #find where data is missing in the gaze position, as probably a blink (or its missing as lost the eye)
            for i in missing_inds:
                iblk_data[i,col] = np.NaN #replace missing data ('.') with NaN
                iblk_data[i,3]   = np.NaN #replace left pupil as NaN (as in a blink)
                iblk_data[i,6]   = np.NaN #replace right pupil as NaN (as in a blink)

        iblk_data = iblk_data.astype(np.float) # convert data from string to floats for computations

        #for binocular data, the shape is:
        # columns: time stamp, left x, left y, left pupil, right x, right y, right pupil
        iblk_trackertime = iblk_data[:,0]
        iblk_lx, iblk_ly, iblk_lp = iblk_data[:,1], iblk_data[:,2], iblk_data[:,3]
        iblk_rx, iblk_ry, iblk_rp = iblk_data[:,4], iblk_data[:,5], iblk_data[:,6]

        # split Efix/Sfix and Esacc/Ssacc into separate lists
        iblk_efix = [iblk_fix[x] for x in range(len(iblk_fix)) if
                     iblk_fix[x][0] == 'EFIX']
        iblk_sfix = [iblk_fix[x] for x in range(len(iblk_fix)) if
                     iblk_fix[x][0] == 'SFIX']

        iblk_ssac = [iblk_sac[x] for x in range(len(iblk_sac)) if
                     iblk_sac[x][0] == 'SSACC']
        iblk_esac = [iblk_sac[x] for x in range(len(iblk_sac)) if
                     iblk_sac[x][0] == 'ESACC']
        iblk_sblk = [iblk_blink[x] for x in range(len(iblk_blink)) if
                     iblk_blink[x][0] == 'SBLINK']
        iblk_eblk = [iblk_blink[x] for x in range(len(iblk_blink)) if
                     iblk_blink[x][0] == 'EBLINK']

        #append to the collection of all data now
        trackertime.append(iblk_trackertime)
        lx.append(iblk_lx)
        ly.append(iblk_ly)
        rx.append(iblk_rx)
        ry.append(iblk_ry)
        lp.append(iblk_lp)
        rp.append(iblk_rp)
        Efix.append(iblk_efix)
        Sfix.append(iblk_sfix)
        Ssac.append(iblk_ssac)
        Esac.append(iblk_esac)
        Sblk.append(iblk_sblk)
        Eblk.append(iblk_eblk)
        Msg.append(iblk_events)

    iblock = {
    'trackertime': [],
    'lx'  : [], 'ly'  : [], 'lp'  : [],
    'rx'  : [], 'ry'  : [], 'rp'  : [],
    'Efix': [], 'Sfix': [],
    'Esac': [], 'Ssac': [],
    'Eblk': [], 'Sblk': [],
    'Msg' : []
    }
    dummy = copy.deepcopy(iblock)
    blocked_data = np.repeat(dummy, nblocks)

    for i in np.arange(nblocks):
        iblock_data = copy.deepcopy(iblock)
        iblock_data['trackertime'] = trackertime[i]
        iblock_data['lx'] = lx[i]
        iblock_data['ly'] = ly[i]
        iblock_data['lp'] = lp[i]
        iblock_data['rx'] = rx[i]
        iblock_data['ry'] = ry[i]
        iblock_data['rp'] = rp[i]
        iblock_data['Efix'] = Efix[i]
        iblock_data['Sfix'] = Sfix[i]
        iblock_data['Esac'] = Esac[i]
        iblock_data['Ssac'] = Ssac[i]
        iblock_data['Eblk'] = Eblk[i]
        iblock_data['Sblk'] = Sblk[i]
        iblock_data['Msg'] = Msg[i]
        blocked_data[i] = iblock_data

    return blocked_data

def _parse_eye_data_trialwise(eye_fname, nblocks, ntrials):

    """
    this function is called by parse_eye_data and will operate on data where the recording was stopped/started for each trial of task data

    """

    d = open(eye_fname, 'r')
    raw_d = d.readlines()
    d.close()

    split_d = []
    for i in range(len(raw_d)):
        tmp = raw_d[i].split()
        split_d.append(tmp)

    #get all lines where 'START'i s seen, as this marks the start of the recording
    start_inds = [x for x in range(len(split_d)) if len(split_d[x]) == 6 and split_d[x][0] == 'START']
    if len(start_inds) != ntrials:
        raise DataError('%d trials are found in the data, not %d as has been input in ntrials' %(len(start_inds),ntrials))

    #get points where recording stopped
    end_inds   = [x for x in range(len(split_d)) if len(split_d[x]) == 7 and split_d[x][0] == 'END']

    if len(start_inds) != len(end_inds):
        raise DataError('the number of times the recording was started and stopped does not align. check problems with acquisition')

    #assign some empty lists to get filled with information
    trackertime = []
    lx   = []; rx   = []
    ly   = []; ry   = []
    lp   = []; rp   = []

    Efix = []; Sfix = []
    Esac = []; Ssac = []
    Eblk = []; Sblk = []
    Msg  = []

    for i in range(len(start_inds)):
        start_line = start_inds[i]
        end_line   = end_inds[i]

        itrl = np.array(split_d[start_line:end_line])

        itrl_event_inds, itrl_events     = [], []
        itrl_blink_inds, itrl_blink      = [], []
        itrl_fix_inds, itrl_fix          = [], []
        itrl_sac_inds, itrl_sac          = [], []
        itrl_input_inds                  = []
        for x,y in np.ndenumerate(itrl):
            if y[0] == 'MSG':
                itrl_event_inds.append(x[0]) # add line index where a message was sent to the eyetracker
                itrl_events.append(y)        # add the line itself to list
            elif y[0] in ['EFIX', 'SFIX']:
                itrl_fix_inds.append(x[0])   # add line index where fixation detected (SR research)
                itrl_fix.append(y)           # add fixation event structure to list
            elif y[0] in ['ESACC', 'SSACC']:
                itrl_sac_inds.append(x[0])   # add line index where saccade detected (SR research)
                itrl_sac.append(y)           # add saccade event structure to list
            elif y[0] in ['EBLINK', 'SBLINK']:
                itrl_blink_inds.append(x[0]) # add line index where blink detected (SR research)
                itrl_blink.append(y)         # add blink event structure to list
            elif y[0] == 'INPUT':
               itrl_input_inds.append(x[0])  # find where 'INPUT' is in data (sometimes appears, has no use...)

        #get all non-data line indices
        itrl_nondata    = sorted(itrl_blink_inds + itrl_sac_inds + itrl_fix_inds + itrl_event_inds + itrl_input_inds)

        itrl_data = np.delete(itrl, itrl_nondata) #remove these lines so all you have is the raw data

        itrl_data = itrl_data[6:] #remove first five lines as these are filler after the recording starts

        try:
            itrl_data = np.vstack(itrl_data) #if this fails its most likely because the end string of dots is missing for some reason
        except ValueError: #if fails, fix
            for item in range(len(itrl_data)):
                itrl_data[item] = itrl_data[item][:7] #make sure each item has only 7 strings. sometimes the eyetracker hasn't appended the random .... at the end

        itrl_data = itrl_data[:,:7]            # only take first 6 columns as these contain data of interest (redundant if above section trips up

        itrl_data = np.vstack(itrl_data) #shape of this should be number of columns in the file (data)

        #before you can convert to float, need to replace missing data where its '.' as nans (in pupil this is '0.0')
        eyecols = [1,2,4,5] #leftx, lefty, rightx, right y col indices
        for col in eyecols:
            missing_inds = np.where(itrl_data[:,col] == '.') #find where data is missing in the gaze position, as probably a blink (or its missing as lost the eye)
            for i in missing_inds:
                itrl_data[i,col] = np.NaN #replace missing data ('.') with NaN
                itrl_data[i,3]   = np.NaN #replace left pupil as NaN (as in a blink)
                itrl_data[i,6]   = np.NaN #replace right pupil as NaN (as in a blink)
        itrl_data = itrl_data.astype(np.float) #convert data from string to floats for computations

        #for binocular data, the shape is:
        # columns: time stamp, left x, left y, left pupil, right x, right y, right pupil
        itrl_trackertime = itrl_data[:,0]
        itrl_lx, itrl_ly, itrl_lp = itrl_data[:,1], itrl_data[:,2], itrl_data[:,3]
        itrl_rx, itrl_ry, itrl_rp = itrl_data[:,4], itrl_data[:,5], itrl_data[:,6]

        # split Efix/Sfix and Esacc/Ssacc into separate lists
        itrl_efix = [itrl_fix[x] for x in range(len(itrl_fix)) if
                     itrl_fix[x][0] == 'EFIX']
        itrl_sfix = [itrl_fix[x] for x in range(len(itrl_fix)) if
                     itrl_fix[x][0] == 'SFIX']

        itrl_ssac = [itrl_sac[x] for x in range(len(itrl_sac)) if
                     itrl_sac[x][0] == 'SSACC']
        itrl_esac = [itrl_sac[x] for x in range(len(itrl_sac)) if
                     itrl_sac[x][0] == 'ESACC']

        itrl_sblk = [itrl_blink[x] for x in range(len(itrl_blink)) if
                     itrl_blink[x][0] == 'SBLINK']
        itrl_eblk = [itrl_blink[x] for x in range(len(itrl_blink)) if
                     itrl_blink[x][0] == 'EBLINK']

        #append to the collection of all data now
        trackertime.append(itrl_trackertime)
        lx.append(itrl_lx)
        ly.append(itrl_ly)
        rx.append(itrl_rx)
        ry.append(itrl_ry)
        lp.append(itrl_lp)
        rp.append(itrl_rp)
        Efix.append(itrl_efix)
        Sfix.append(itrl_sfix)
        Ssac.append(itrl_ssac)
        Esac.append(itrl_esac)
        Sblk.append(itrl_sblk)
        Eblk.append(itrl_eblk)
        Msg.append(itrl_events)

    trialsperblock = ntrials/nblocks
    rep_inds  = np.repeat(np.arange(nblocks),trialsperblock)
    #plt.hist(rep_inds, bins = nblocks) # all bars should be the same height now if its work? @ trials per block (80)

    iblock = {
    'trackertime': [],
    'lx'  : [], 'ly'  : [], 'lp'  : [],
    'rx'  : [], 'ry'  : [], 'rp'  : [],
    'Efix': [], 'Sfix': [],
    'Esac': [], 'Ssac': [],
    'Eblk': [], 'Sblk': [],
    'Msg' : []
    }
    dummy = copy.deepcopy(iblock)
    blocked_data = np.repeat(dummy, nblocks)

    for i in np.arange(nblocks):
        inds = np.squeeze(np.where(rep_inds == i))
        iblock_data = copy.deepcopy(iblock)
        for ind in inds: #add trialwise info into the blocked structure, continuous data rather than sectioned
            iblock_data['trackertime'].append(trackertime[ind])
            iblock_data['lx'].append(lx[ind])
            iblock_data['ly'].append(ly[ind])
            iblock_data['lp'].append(lp[ind])
            iblock_data['rx'].append(rx[ind])
            iblock_data['ry'].append(ry[ind])
            iblock_data['rp'].append(rp[ind])
            iblock_data['Efix'].append(Efix[ind])
            iblock_data['Sfix'].append(Sfix[ind])
            iblock_data['Esac'].append(Esac[ind])
            iblock_data['Ssac'].append(Ssac[ind])
            iblock_data['Eblk'].append(Eblk[ind])
            iblock_data['Sblk'].append(Sblk[ind])
            iblock_data['Msg'].append(Msg[ind])
            blocked_data[i] = iblock_data
    for block in range(len(blocked_data)): #concatenate trialwise signals into whole block traces to make artefact removal easier
        blocked_data[block]['trackertime'] = np.hstack(blocked_data[block]['trackertime'])
        blocked_data[block]['lx']          = np.hstack(blocked_data[block]['lx']         )
        blocked_data[block]['ly']          = np.hstack(blocked_data[block]['ly']         )
        blocked_data[block]['lp']          = np.hstack(blocked_data[block]['lp']         )
        blocked_data[block]['rx']          = np.hstack(blocked_data[block]['rx']         )
        blocked_data[block]['ry']          = np.hstack(blocked_data[block]['ry']         )
        blocked_data[block]['rp']          = np.hstack(blocked_data[block]['rp']         )
    return blocked_data

def find_missing_periods(data, nblocks):
    '''
    this function will find all sections of missing data. this typically relates to blinks
    but it will also find small patches where an eye has been dropped from the data.
    this just finds all patches, no matter the size, that are missing and outputs meaningful info into your data structure

    data argument expects a list, where each item in the list is a dictionary containing the data for a continuous block.

    binocular recorded data is expected here, where in each block the following keys are in the data dict:
        lx, ly, rx, ry.
    these correspond to the left and right gaze values. this function will iterate over each one to identify missing periods
    in each signal, and then output this information into new keys in your data:
    Eblk_lx, Eblk_ly, Eblk_rx, Eblk_ry
    '''

    if not isinstance(data, np.ndarray):
        raise Exception('check the format of your data. an array of dictionaries for each block is expected')

    if nblocks != len(data): #len(data) should give you the number of blocks in the data file
        raise Exception('there are not as many blocks in the data as you think. check this!')

    for block in data: #iterate over each block of data
        if 'lx' not in block.keys():
            raise Exception('A signal relating to the left eye is missing. Make sure the left x is labelled \'lx\'')
        if 'ly' not in block.keys():
            raise Exception('A signal relating to the left eye is missing. Make sure the left y is labelled \'ly\'')
        if 'rx' not in block.keys():
            raise Exception('A signal relating to the right eye is missing. Make sure the right x is labelled \'rx\'')
        if 'ry' not in block.keys():
            raise Exception('A signal relating to the right eye is missing. Make sure the right y is labelled \'ry\'')

        s_lx = []; s_rx = [];
        s_ly = []; s_ry = [];
        e_lx = []; e_rx = [];
        e_ly = []; e_ry = [];

        #find missing data in each gaze trace (left x & y, right x & y) separately for precision
        mlx = np.array(np.isnan(block['lx']) == True, dtype = int) # array of 1's and 0's for if data is missing at that time point
        dlx = np.diff(mlx) #find change-points, +1 means goes from present to absent, -1 from absent to present
        s_lx = np.squeeze(np.where(dlx ==  1)) #find points where it starts to be missing
        e_lx = np.squeeze(np.where(dlx == -1))+1 #find points where it stops being missing

        mly = np.array(np.isnan(block['ly']) == True, dtype = int)
        dly = np.diff(mly)
        s_ly = np.squeeze(np.where(dly ==  1))
        e_ly = np.squeeze(np.where(dly == -1))+1

        mrx = np.array(np.isnan(block['rx']) == True, dtype = int)
        drx = np.diff(mrx)
        s_rx = np.squeeze(np.where(drx ==  1))
        e_rx = np.squeeze(np.where(drx == -1))+1

        mry = np.array(np.isnan(block['ry']) == True, dtype = int)
        dry = np.diff(mry)
        s_ry = np.squeeze(np.where(dry ==  1))
        e_ry = np.squeeze(np.where(dry == -1))+1

        Eblk_lx = []; Eblk_ly = []
        Eblk_rx = []; Eblk_ry = []
        #left x
        for i in range(s_lx.size):
            if s_lx.size == 1:
                start = s_lx.tolist()
                end   = e_lx.tolist()
            else:
                start = s_lx[i]
                if i < e_lx.size: #check within range of the number of missing periods in data
                    end = e_lx[i]
                elif i == e_lx.size:
                    end = e_lx[-1]
                else:
                    end = e_lx[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
                #create a blink event structure:
                # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['LX_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_lx.append(evnt)
        #left y
        for i in range(s_ly.size):
            if s_ly.size == 1:
                start = s_ly.tolist()
                end   = e_ly.tolist()
            else:
                start = s_ly[i]
                if i < e_ly.size: #check within range of the number of missing periods in data
                    end = e_ly[i]
                elif i == e_ly.size:
                    end = e_ly[-1]
                else:
                    end = e_ly[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['LY_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_ly.append(evnt)
        #right x
        for i in range(s_rx.size):
            if s_rx.size == 1:
                start = s_rx.tolist()
                end   = e_rx.tolist()
            else:
                start = s_rx[i]
                if i < e_rx.size: #check within range of the number of missing periods in data
                    end = e_rx[i]
                elif i == e_rx.size:
                    end = e_rx[-1]
                else:
                    end = e_rx[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['RX_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_rx.append(evnt)
        #right y
        for i in range(s_ry.size):
            if s_ry.size == 1:
                start = s_ry.tolist()
                end   = e_ry.tolist()

            else:
                start = s_ry[i]
                if i < e_ry.size: #check within range of the number of missing periods in data
                    end = e_ry[i]
                elif i == e_ry.size:
                    end = e_ry[-1]
                else:
                    end = e_ry[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['RY_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_ry.append(evnt)
        #append these new structures to the dataset
        block['Eblk_lx'] = Eblk_lx
        block['Eblk_rx'] = Eblk_rx
        block['Eblk_ly'] = Eblk_ly
        block['Eblk_ry'] = Eblk_ry

    return data

def interpolateBlinks_Blocked(block, trace):

    """
    - This function will interpolate the blinks in eyelink data that is in blocks (longer continuous segments)
    - The data that it will handle can either be segmented trials (due to trialwise stop/start of recording in the task) stitched back together
    - or just continuous block data, depending on the structure you give it.

    - If you use the script AttSacc_ParseData.py to parse the data, and are running AttSacc_CleanBlockedData.py, then the block structure should be suitable for this function.

    block argument will expect a dictionary that has the following fields:
    - trackertime: the timeseries of timepoints defined by the eyelink time, rather than trial time
    - lx      - timeseries of the x value for the left eye
    - ly      - timeseries of the y value for the left eye
    - rx      - timeseries of the x value for the right eye
    - ry      - timeseries of the y value for the right eye

    - Eblk_lx - list of events characterising missing periods of data in lx
    - Eblk_rx - list of events characterising missing periods of data in rx
    - Eblk_ly - list of events characterising missing periods of data in ly
    - Eblk_ry - list of events characterising missing periods of data in ry
    ----------- these blink structures have the following format:
    ----------- ['Event_code', start_blocktime, end_blocktime, start_trackertime, end_trackertime, duration]
    ----------- block code e.g. 'LX_BLK'

    trace argument expects a string specifying what trace you want to clean (e.g. 'lx' for left eye, x signal)

    Missing periods of data will be removed in the following way:

    periods of missing data of under 10 samples will be linearly interpolated within a window of 10 samples either side of the start and end of the period
    """
    if not isinstance(block, dict):
        raise Exception('data supplied is not a dictionary')

    if not isinstance(trace, str):
        raise Exception('the trace indicator supplied is not a string. \n please input a string e.g. \'lx\'')

    if trace not in block.keys():
        raise Exception('the signal you want to clean does not exist in the data. check your trace labels in your data')

    signal = block[trace] #extract the signal that needs cleaning

    eventlabel = 'Eblk_%s'%trace

    if eventlabel not in block.keys():
        raise Exception('the missing period information for this signal is not in your data structure')

    blinks = np.array(block[eventlabel])       #get the desired blink structures
    if blinks.size != 0:
        blinks = blinks[:,1:]            #remove the first column as it's a string for the event code, not needed now
        blinks = blinks.astype(float).astype(int) # change the strings to integers. need to go via float or it fails.

        short_duration_inds  = np.where(np.in1d(blinks[:,4], range(21)))[0]    # find the blinks that are below 20 samples long
        medium_duration_inds = np.where(np.in1d(blinks[:,4], range(21,51)))[0] # find the blinks that are between 21 and 50 samples long
        long_duration_inds   = np.where(blinks[:,4] > 50)[0]                   # find blinks that are over 50 samples long

        short_blinks  = blinks[short_duration_inds,:]
        medium_blinks = blinks[medium_duration_inds,:]
        long_blinks   = blinks[long_duration_inds,:]

        #linear interpolate across these smaller periods before proceeding.
        for blink in short_blinks:
            start, end               = blink[0], blink[1] #get start and end periods of the missing data
            to_interp                = np.arange(start, end) #get all time points to be interpolated over

            #set up linear interpolation
            inttime                  = np.array([start,end])
            inttrace                 = np.array([signal[start], signal[end]])
            fx_lin                   = sp.interpolate.interp1d(inttime, inttrace, kind = 'linear')

            interptrace              = fx_lin(to_interp)
            signal[to_interp] = interptrace
        for blink in medium_blinks:
            start, end               = blink[0], blink[1] #get start and end periods of the missing data
            to_interp                = np.arange(start, end) #get all time points to be interpolated over

            #set up linear interpolation
            inttime                  = np.array([start,end])
            inttrace                 = np.array([signal[start], signal[end]])
            fx_lin                   = sp.interpolate.interp1d(inttime, inttrace, kind = 'linear')

            interptrace              = fx_lin(to_interp)
            signal[to_interp] = interptrace

        #now cubic spline interpolate across the larger missing periods (blinks)
        for blink in long_blinks:
            start, end            = blink[0], blink[1] #get start and end of these missing samples
            if end+40 >= signal.size: #this blink happens just before the end of the block, so need to adjust the window
                window            = [start-80, start - 40, end, signal.size-1] #reduce the window size but still cubic spline
            elif end+40 <= signal.size and end+80 >= signal.size:
                window            = [start-80, start-40, end+40, signal.size-1]
            else:
                window            = [start-80, start-40, end+40, end+80] #set the window for the interpolation
            inttime               = np.array(window)
            if end + 40 >= signal.size:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40])                 , np.nanmedian(signal[start-40:start-1]),
                                              np.nanmedian(signal[end:int(np.floor((signal.size-1-end)/2))]), np.nanmedian(signal[int(np.ceil((signal.size-1-end)/2)):signal.size-1]) ])
            elif end+40 <= signal.size and end + 80 >= signal.size:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40])                 , np.nanmedian(signal[start-40:start-1]),
                                              np.nanmedian(signal[end:end+40]), np.nanmedian(signal[end+40:signal.size-1]) ])
            else:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40]), np.nanmedian(signal[start-40:start-1]), # by giving the nanmedian between these points,
                                              np.nanmedian(signal[end+1:end+40])     , np.nanmedian(signal[end+40:end+80]) ])  # points, it accounts for variance of the signal
            fx_cub                = sp.interpolate.interp1d(inttime, inttrace, kind = 'cubic')

            if end+30 >= signal.size:
                to_interp         = np.arange(start-30, signal.size-1)
            else:
                to_interp         = np.arange(start-30, end+30) #interpolate just outside the start of the missing period, for cases of large changes due to blinks
            interptrace           = fx_cub(to_interp)
            signal[to_interp]     = interptrace
    #output the data into the block structure
    block[trace] = signal
    return block