from unittest import TestCase
import glob

from lxml import etree

class ValidationError(Exception):
    pass

class TestSampleFileValidation(TestCase):
    def test_ukrdc_sample_files(self):
        # For each sample file
        for sample_path in glob.glob("sample_files/ukrdc/*.xml"):
            # Run as a subtest
            with self.subTest(msg=sample_path):
                # Open the schema and sample files for reading
                with open(
                    "schema/ukrdc/UKRDC.xsd", "r", encoding="utf-8"
                ) as schema_file, open(
                    sample_path, "r", encoding="utf-8"
                ) as sample_file:

                    # Create a new schema object to track errors for this file
                    xml_schema = etree.XMLSchema(
                        etree.parse(
                            schema_file,
                            parser=None,
                        )
                    )

                    # Try validating the sample file against the schema
                    try:
                        xml_doc = etree.parse(sample_file, None)
                        xml_schema.assertValid(xml_doc)
                    # Initially catch errors to allow reporting multiple issues in one file
                    except etree.DocumentInvalid as e:
                        tree = etree.ElementTree(xml_doc.getroot())
                        # Print all errors
                        print("Validation error(s):")
                        for error in xml_schema.error_log:
                            print("  Line {}: {}".format(error.line, error.message))

                            for e in tree.xpath(".//*"):
                                if error.line == e.sourceline:
                                    xml_path = tree.getpath(e)
                                    print(xml_path)
                                    break

                        # Raise an exception to fail the test and report the full error list
                        raise ValidationError(
                            f"{len(xml_schema.error_log)} validation error(s) in {sample_path}. See full output above for details."
                        ) from e


from urllib.parse import quote_plus
from bs4 import BeautifulSoup
#selenium : web test에 사용되는 프레임워크, webdriver API를 통해 렌더링이 완료된 후의 DOM 결과물에 접근할 수 있음(브라우저 제어가 필요)
#pip install selenium
#직접 브라우저를 제어하기 때문에 header값 없이도 크롤링이 가능
#여기선 Chrome 사용 webdriver 설치 : https://chromedriver.chromium.org/downloads
from selenium import webdriver

baseUrl = 'https://www.google.com/search?q='
plusUrl = input('검색어 입력 : ')
resultUrl = baseUrl + quote_plus(plusUrl)

#chrome webDriver 위치가 현재 개발폴더 위치와 다르면 Chrome({경로})와 같이 사용
driver = webdriver.Chrome()

#브라우저가 열리고 입력된 url로 이동
driver.get(resultUrl)

html = driver.page_source
soup = BeautifulSoup(html)

#select로 가져올 경우 list형식으로 가져옴
r = soup.select('.r')
for i in r:
    #list object의 경우엔 text를 가져올 수 없음, 텍스트를 불러오기 위해 select_on 사용
    print(i.select_one('.LC20lb.DKV0Md').text)
    #print(i.select_one('.iUh30.bc').text)
    print(i.a.attrs['href'], '\n')

driver.close()


"""
-*- coding: utf-8 -*-
@author: socratio
@inspiration: drew original inspiration from cleartonic twitchtriviabot. Almost nothing left in this code from that project.

"""
import json
from twitchio import websocket
from twitchio.ext import commands
import yaml
import asyncio
import os
import random

class ChatBot(commands.Bot):

    def __init__(self):
        #load the auth and connect to twitch
        with open(os.path.join(os.getcwd(),'config','auth_config.yml')) as auth:
            self.auth = yaml.safe_load(auth)
        super().__init__(irc_token=f"{self.auth['pass']}", client_id='...', nick=f"{self.auth['nick']}", prefix='!',initial_channels=[f"{self.auth['chan']}"])

        #load the trivia configuration
        with open(os.path.join(os.getcwd(),'config','trivia_config.yml')) as config:
            self.trivia_config = yaml.safe_load(config)

        #create admins array, empty players and questions arrays, boolean variables, and empty answer messages object
        self.admins = [i.strip() for i in self.trivia_config['admins'].split(",")]
        self.players = []
        self.questionlist = []
        self.active_game = False
        self.questionisactive = False
        self.active_question = False
        self.scoringopen = False
        self.answermessages = {}

        #load the scoreboard, set the list of past winners, increment the game number
        self.refresh_scores()
        try:
            self.pastwinners = self.scores[f'Season {self.trivia_config["season"]}']['shirtwinners']
        except:
            self.scores[f'Season {self.trivia_config["season"]}'] = {"gamesplayed":0, "shirtwinners":[], "scoreboard":{}}
            self.pastwinners = self.scores[f'Season {self.trivia_config["season"]}']['shirtwinners']
        self.game_number = self.scores[f'Season {self.trivia_config["season"]}']['gamesplayed']+1

        #load the questions and populate the questions array
        with open(os.path.join(os.getcwd(),'config','triviaset.json')) as self.questions:
            self.questions = json.load(self.questions)
        for question in self.questions.items():
            self.questionlist.append(Question(question))

        #populate the players array
        for player in self.scores[f'Season {self.trivia_config["season"]}']['scoreboard'].items():
            self.players.append(Player(player))

    #updates the scoreboard dict object
    def refresh_scores(self):
        with open(os.path.join(os.getcwd(),'config','scores',"scoreboard.json")) as scores:
            self.scores = json.load(scores)

    #clears json of scores for this game, sorts and adds scores back to json, resulting in sorted scores every time. Also saves scores to scoreboard file
    def commit_scores(self):
        self.scores[f'Season {self.trivia_config["season"]}'][f'Game {self.game_number}'] = {}
        self.scores[f'Season {self.trivia_config["season"]}']['scoreboard'] = {}
        for player in sorted(self.players, key=lambda player:player.seasonpoints, reverse=True):
            self.scores[f'Season {self.trivia_config["season"]}']['scoreboard'][player.name] = player.seasonpoints
        for player in sorted(self.players, key=lambda player:player.gamepoints, reverse=True):
            self.scores[f'Season {self.trivia_config["season"]}'][f'Game {self.game_number}'][player.name] = player.gamepoints
        with open(os.path.join(os.getcwd(),'config','scores',"scoreboard.json"),'w') as outfile:
            json.dump(self.scores, outfile, indent=4)

    #Broadcast ready state to twitch channel
    async def event_ready(self):
        print(f'Ready | {self.nick}')
        ws = bot._ws
        await ws.send_privmsg(self.initial_channels[0],"I have indeed been uploaded, sir.")

    #major message reading function
    async def event_message(self, message):
        if message.author != self.nick:
            print(f'{message.author.name}: {message.content}')
            await self.handle_commands(message)
            if self.scoringopen == True and not message.content.startswith('!'):
                if message.author.name in self.answermessages:
                    del self.answermessages[message.author.name]
                self.answermessages[message.author.name] = message.content

    @commands.command(name='test')
    async def test(self, ctx):
        await ctx.send(f'Hello {ctx.author.name}!')

    #TRIVIA COMMANDS AND PROCEDURES
    @commands.command(name='start')
    #!Start command starts the trivia game
    async def start(self, ctx):
        if ctx.author.name in self.admins and not self.active_game:
            self.active_game = True
            print('Starting Game.')
            await ctx.send("Game starts in 15 seconds. Watch the chat for the question. Good luck!")
            await asyncio.sleep(15)
            if self.active_game:
                await self.callquestion()

    @commands.command(name='next')
    #!next starts the process of asking the next question after 10 seconds and scoring after 20 seconds
    async def nextq(self, ctx):
        if ctx.author.name in self.admins and not self.questionisactive:
            self.questionisactive = True
            print('Received call for next question.')
            await ctx.send("Next question coming in 10 seconds. Keep an eye on the chat!")
            await asyncio.sleep(10)
            if self.active_game:
                await self.callquestion()
        else:
            print('Received call for next question, but an active question exists or it is not an admin. Ignoring call.')

    @commands.command(name='end')
    #!end ends this game of trivia, commits scores to json, and refreshes the scores
    async def endtrivia(self, ctx):
        if ctx.author.name in self.admins and self.active_game:
            print("Ending game.")
            self.scoringopen = False
            self.active_game = False
            self.active_question = False
            if any(i.gamepoints > 0 for i in self.players):
                for player in sorted(self.players, key=lambda x:x.gamepoints, reverse=True):
                    if player.name not in self.scores[f'Season {self.trivia_config["season"]}']['shirtwinners']:
                        self.scores[f'Season {self.trivia_config["season"]}']['shirtwinners'].append(player.name)
                        self.pastwinners.append(player.name)
                        break
                self.scores[f'Season {self.trivia_config["season"]}']['gamesplayed'] = self.game_number
                await ctx.send(f"Ending this game of trivia. Congratulations to {self.pastwinners[-1]} on the new shirt! I hope everyone had fun!")
                self.commit_scores()
                self.refresh_scores()

    @commands.command(name='bonus')
    #!bonus reads the message, finds the user targeted for bonus points, finds the point value of the bonus, assigns the extra points if the player exists or creates them if not, and refreshes the scores
    async def bonus(self, ctx):
        if ctx.author.name in self.admins:
            print(f"Received call for bonus points from {ctx.author.name}.")
            bonustarget = ctx.message.content.split()[1].lower()
            bonuspoints = int(ctx.message.content.split()[2])
            if any(bonustarget == player.name for player in self.players):
                for player in self.players:
                    if player.name == bonustarget:
                        player.gamepoints += int(bonuspoints)
                        returnstr = player.gamepoints
            else:
                print(f'Player {bonustarget} does not exist. Creating.')
                user = Player(bonustarget,bonuspoints)
                self.players.append(user)
            self.commit_scores()
            self.refresh_scores()
            await ctx.send(f'Player {bonustarget} received {bonuspoints} bonus points. Their new total is {returnstr} points.')

    @commands.command(name='lasttop5')
    #!lasttop5 calls the top 5 scores from the last game played
    async def lasttop5(self, ctx):
        if ctx.author.name in self.admins:
            returnstr = "TOP 5 SCORES FOR THE LAST GAME:\t"
            lastgameno = self.scores[f'Season {self.trivia_config["season"]}']['gamesplayed']
            lastgamescores = self.scores[f'Season {self.trivia_config["season"]}'][f'Game {lastgameno}']
            for score in sorted(lastgamescores.items(), key=lambda x:x[1], reverse=True)[:5]:
                returnstr += f"{score[0]}: {score[1]}  "
            await ctx.send(returnstr)

    async def callquestion(self):
        self.active_question = self.questionlist.pop(0)
        self.scoringopen = True
        self.answermessages = {}
        ws = bot._ws
        await ws.send_privmsg(self.initial_channels[0],f"Question {self.active_question.questionno}: {self.active_question.question}")
        await asyncio.sleep(20)
        self.scoringopen = False
        await self.scorequestion()
        self.questionisactive = False

    async def scorequestion(self):
        self.scoringopen = False
        ws = bot._ws
        self.point_dict = {}
        returnstr = f"The answer was **{self.active_question.answers[0]}**.\t"
        #check that all players that answered exist as Player objects
        for name in self.answermessages.keys():
            if not any(player.name == name for player in self.players):
                print(f'Player {name} does not exist. Creating.')
                user = Player(name)
                self.players.append(user)

        #find all the correct answers, building the list of points as it goes
        for answer in self.answermessages.items():
            for proof in self.active_question.answers:
                if answer[1].lower() == proof.lower():
                    self.point_dict[answer[0]] = 0
                    break
            else:
                with open(os.path.join(os.getcwd(),"config","aliases.json")) as aliases:
                    aliases = json.load(aliases)
                for name in aliases.items():
                    if answer[1].lower() in name[1] and name[0] == self.active_question.answers[0]:
                        self.point_dict[answer[0]] = 0
            for proof in self.active_question.deepcut:
                if answer[1].lower() == proof.lower():
                    self.point_dict[answer[0]] = 3

        #check if only 1 person answered, if so, award 3 bonus points
        for name,points in self.point_dict.items():
            if len(self.point_dict) == 1:
                self.point_dict[name] += 3
            if 1 < len(self.point_dict) < 4:
                self.point_dict[name] += 1

        #award 1 point for everyone, an extra point for the first 14, and another point for the first 6
        idx = 0
        for name,points in self.point_dict.items():
            if idx == 0:
                returnstr += f"{name} was the first to answer correctly."
            if idx < 6:
                self.point_dict[name] += 1
            if idx < 20:
                self.point_dict[name] += 1
            self.point_dict[name] += 1
            idx += 1
            #update the player object with the new points
            for player in self.players:
                if player.name == name:
                    player.gamepoints += self.point_dict[name]
                    player.seasonpoints += self.point_dict[name]
        self.commit_scores()
        await ws.send_privmsg(self.initial_channels[0],returnstr)

    #CHAT RESPONSES AND COMMAND FUNCTIONS
    @commands.command(name='score')
    #!score finds the score of the user sending the message and sends it in chat
    async def score(self, ctx):
        print(f'Received a score check for {ctx.author.name}')
        if any(player.name == ctx.author.name for player in self.players):
            for player in self.players:
                if player.name == ctx.author.name:
                    print(f'Found player {player.name} with {player.gamepoints} game points and {player.seasonpoints} season points.')
                    user = player
                    if self.active_game:
                        scorestr = f"User {player.name} has {player.gamepoints} points in this game and {player.seasonpoints} for the season."
                    else:
                        scorestr = f"User {player.name} has {player.seasonpoints} points in this season."
                    break
        else:
            print(f'Player {ctx.author.name} does not exist. Creating.')
            user = Player(ctx.author.name)
            self.players.append(user)
            scorestr = f"User {user.name} has 0 points. Welcome to trivia!"
        await ctx.send(scorestr)

    @commands.command(name='raffle')
    #!raffle finds the raffle ticket count of the user sending the message and sends it in chat
    async def raffle(self, ctx):
        print(f'Received a raffle check for {ctx.author.name}')
        if any(player.name == ctx.author.name for player in self.players):
            for player in self.players:
                if player.name == ctx.author.name:
                    rafflecount = int(player.seasonpoints/30)
                    print(f'Found player {player.name} with {player.gamepoints} game points, {player.seasonpoints} season points, and {rafflecount} raffle tickets.')
                    user = player
                    if not self.active_game:
                        scorestr = f"User {player.name} has {player.seasonpoints} for the season resulting in {rafflecount} raffle tickets."
                    break
        else:
            print(f'Player {ctx.author.name} does not exist. Creating.')
            user = Player(ctx.author.name)
            self.players.append(user)
            scorestr = f"User {user.name} has 0 points and no raffle tickets. Welcome to trivia!"
        await ctx.send(scorestr)

    @commands.command(name='top5')
    #!top5 returns the top5 scores for the game if a game is active or for the season if a game is not active
    async def top5(self, ctx):
        if ctx.author.name in self.admins:
            returnstr = 'TOP 5: '
            print(f'Received top 5 check from {ctx.author.name}.')
            if self.active_game:
                self.refresh_scores()
                for i in sorted(self.players, key=lambda player:player.gamepoints, reverse=True)[:5]:
                    returnstr += (f'{i.name}: {i.gamepoints}\t')
            else:
                returnstr = "THIS SEASON'S TOP 5:  "
                for i in sorted(self.players, key=lambda player:player.seasonpoints, reverse=True)[:5]:
                    returnstr += (f'{i.name}: {i.seasonpoints}\t')
            await ctx.send(returnstr)

    @commands.command(name='topless')
    #!topless returns the top 5 player scores for players who have not yet won a shirt as defined in pastwinners
    async def topless(self, ctx):
        if ctx.author.name in self.admins:
            returnstr = 'TOP 5 SHIRTLESS THIS '
            self.topless = []
            print(f'Received top 5 shirtless check from {ctx.author.name}.')
            self.refresh_scores()
            if self.active_game:
                returnstr += 'GAME: '
                for player in sorted(self.players, key=lambda x:x.gamepoints, reverse=True):
                    if player.name not in self.pastwinners and len(self.topless) < 5:
                        self.topless.append(player)
                        returnstr += f'{player.name}: {player.gamepoints}  '
                    else:
                        continue
            else:
                returnstr += 'SEASON: '
                for player in self.scores[f'Season {self.trivia_config["season"]}']['scoreboard'].items():
                    if player[0] not in self.pastwinners and len(self.topless) < 5:
                        self.topless.append(player[0])
                        returnstr += f'{player[0]}: {player[1]}  '
                    else:
                        continue
            await ctx.send(returnstr)

    @commands.command(name='stop')
    #!stop forces the chatbot to shut down
    async def stop(self, ctx):
        if ctx.author.name in self.admins:
            print(f'Received stop command from {ctx.author.name}.')
            if self.active_game:
                self.active_game = False
            await ctx.send('I have been commanded to stop. The Vision trivia bot is shutting down. See you next time!')
            await bot._ws.teardown()

    @commands.command(name='rafflewinner')
    #!rafflewinner generates a list of raffle tickets based on a person's total points/30 and selects a random winner
    async def rafflewinner(self, ctx):
        if ctx.author.name in self.admins:
            await ctx.send('This is the moment you have ALL been waiting for. The winner of the biggest prize in Stranded Panda Trivia history is...*shuffles raffle tickets for 10 seconds*')
            await asyncio.sleep(10)
            self.refresh_scores()
            with open(os.path.join(os.getcwd(),'config','scores',"scoreboard.json")) as scoreboard:
                scoreboard = json.load(scoreboard)
            scoreboard = scoreboard[f'Season {self.trivia_config["season"]}']['scoreboard']
            rafflelist = []
            for player in scoreboard.items():
                ticketcount = int(player[1]/30)
                for count in range(0,ticketcount):
                    rafflelist.append(player[0])
            drawingwinner = random.choice(rafflelist)
            await ctx.send("The hosts now have the raffle winner in their debatably capable hands...")
            print(f'The raffle winner is {drawingwinner}')

    @commands.command(name='seasonwinner')
    #!seasonwinner takes the top 14 scores for the season, adds them together, and produces the top 10
    async def seasonwinner(self, ctx):
        if ctx.author.name in self.admins:
            returnstr = "This season's top 10: "
            scorelists = {}
            sortedlists = {}
            finalscores = {}
            with open(os.path.join(os.getcwd(),'config','scores',"scoreboard.json")) as scoreboard:
                scoreboard = json.load(scoreboard)
            for game in scoreboard[f'Season {self.trivia_config["season"]}'].items():
                if (game[0].startswith("Game ")):
                    for player in game[1].items():
                        if player[0] not in scorelists:
                            scorelists[f'{player[0]}'] = []
                        scorelists[f'{player[0]}'].append(player[1])
            for scores in scorelists.items():
                sortedlists[f'{scores[0]}'] = sorted(scores[1],reverse=True)
            for player in sortedlists.items():
                finalscores[f'{player[0]}'] = sum(player[1][0:14])
            scoreboard = {}
            for player in sorted(finalscores.items(), key=lambda player:player[1], reverse=True):
                scoreboard[player[0]] = player[1]
            for score in sorted(scoreboard.items(), key=lambda x:x[1], reverse=True)[:10]:
                returnstr += f"{score[0]}: {score[1]}  "
            overallwinner = sorted(scoreboard.items(), key=lambda x:x[1], reverse=True)[0]
            await ctx.send("Calculating the season's winner...removing the bottom 2 scores...swapping the bonus Halloween week...")
            await asyncio.sleep(5)
            await ctx.send(f'The winner of this season of Stranded Panda Twitch Trivia is... {overallwinner[0]} with {overallwinner[1]} points!!! Congratulations {overallwinner[0]}!!!')
            await asyncio.sleep(5)
            await ctx.send(returnstr)

    @commands.command(name='rescore')
    #!rescore removes the most recently awarded points and rescores using the most recently submitted answer list.
    async def rescore(self, ctx):
        if ctx.author.name in self.admins and not self.questionisactive and not self.scoringopen:
            print(f"Received call for rescore from {ctx.author.name}.")
                    #update the player objects with the new points
            for name,points in self.point_dict.items():
                for player in self.players:
                    if player.name == name:
                        player.gamepoints -= points
                        player.seasonpoints -= points
            self.commit_scores()
            await self.scorequestion()
            await ctx.send("Rescoring complete.")

class Question(object):
    #Each question will be an object to be added to a list of objects
    def __init__(self, question):
        badap = 'â€™'
        str_ap = "'"
        self.question = str(question[1]['Question'].replace(badap,str_ap))
        self.answers = question[1]['Answers']
        self.deepcut = question[1]['DeepCut']
        self.questionno = question[0]

class Player(object):
    #This establishes players in the current game
    def __init__(self,playername, pointstart=0):
        #if the playername variable is not a string, it's going to be a dictionary object with existing points totals.
        #The playername variable will be a string if coming from a !score command and a dictionary object if coming from bot initialization
        if not isinstance(playername, str):
            self.seasonpoints = playername[1]
            self.name = playername[0]
        else:
            self.seasonpoints = 0
            self.name = playername
        self.gamepoints = pointstart

if __name__ == '__main__':
    bot = ChatBot()
    bot.run()


from django.conf.urls import url
from . import views

from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    url(r'^$', views.first_view, name='first_view'),
    url(r'^uimage/$', views.uimage, name='uimage'),
    url(r'^dface/$', views.dface, name='dface'),
    url(r'^crop/$', views.crop, name='crop'),
    url(r'^backgroundsubtract/$', views.backgroundsubtract, name='backgroundsubtract'),
    url(r'^binarize/$', views.binarize, name='binarize'),
    url(r'^webcam/$', views.webcam, name='webcam'),
    url(r'^stream/$', views.stream, name='stream'),
    url(r'^capture/$', views.capture, name='capture'),
]

urlpatterns += static(settings.MEDIA_URL,
                      document_root=settings.MEDIA_ROOT)


from flask import Flask
from flask_cors import CORS
from flask_marshmallow import Marshmallow
from config import config
from .main import main as main_blueprint

'''
Application factory for application package. \
Delays creation of an app by moving it into a factory function that can be \
explicitly invoked from script and apply configuration changes.
'''

cors = CORS(
    main_blueprint,
    origins=['http://127.0.0.1:4200', 'http://localhost:4200'],
    supports_credentials=True
)

ma = Marshmallow()

def create_app(config_name):
    app = Flask(__name__)
    # Importing configuration settings directly into app
    app.config.from_object(config[config_name])
    config[config_name].init_app(app)

    # Initializing extensions after app is created
    cors.init_app(app)
    ma.init_app(app)

    # Manually creating app_context to access objects outside of view functions
    with app.app_context():
        app.register_blueprint(main_blueprint, url_prefix='/daron')

    return app


import sqlite3, sys
from pathlib import Path
from . import Notes
from tqdm import tqdm

db_path = "database/xhs_tesla_notes.db"

def fill(db_path=db_path):
    blank_query = "SELECT COUNT(*) FROM notes WHERE content is ''"
    try:
        conn = sqlite3.connect(db_path)
        cursor = conn.cursor()

        amount = cursor.execute("SELECT COUNT(*) FROM notes").fetchone()[0]
        blank_amount = cursor.execute(blank_query).fetchone()[0]
        print(
            f"There are {amount} notes in the database, {blank_amount} of them have blank content, blank rate is {blank_amount/amount}"
        )

        blank_notes_query = cursor.execute(blank_query)
        notes_null_content = blank_notes_query.fetchall()
        # notes_null_content = []

        for note in tqdm(notes_null_content):
            try:
                null_con = Notes.Note(
                    note[0], note[1], note[2], note[3], note[4], note[5], note[10]
                )
                print(f'Filling content for note with id {null_con.id}')
                content = null_con.get_content()
                # print(f"Obtained content: {content}")

                update_query = "UPDATE notes SET content = ? WHERE id = ?"
                cursor.execute(update_query, (content, null_con.id))
                conn.commit()
                # print(f'Successfully filled content for note with id {null_con.id}')

            except Exception as inner_exc:
                print(f"Error processing note with id {note[0]}: {inner_exc}")

        New_blank_amount = cursor.execute(blank_query).fetchone()[0]
        print(
            f"{blank_amount-New_blank_amount} of them have been filled, blank_rate now is {New_blank_amount/amount} as {New_blank_amount} of {amount}"
        )

    except Exception as outer_exc:
        print(f"Error connecting to the database: {outer_exc}")

    finally:
        conn.commit()
        conn.close()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('spurt', '0020_linkpost_scrape_token'),
    ]

    operations = [
        migrations.RenameField(
            model_name='linkpost',
            old_name='pub_date',
            new_name='scraped_pub_date',
        ),
    ]


import sys
import logging
import click
import os
sys.path.append('.')
from src.classes import Dataset

logger = logging.getLogger(__name__)

@click.command()
@click.option(
    '--n_images', default=10,
    help="Number of images per tissue"
)
@click.option(
    '--n_tissues', default=6,
    help="Number of tissues with most numbers of samples"
)
def main(n_images, n_tissues):
    os.makedirs('data/patches', exist_ok=True)
    logger.info('Initializing patches script')
    dataset = Dataset(n_images=n_images, n_tissues=n_tissues)
    dataset.get_patchcoordfiles()

if __name__ == '__main__':
    logging.basicConfig(
        filename='logs/patches.log', level=logging.DEBUG,
        format=(
            "%(asctime)s | %(name)s | %(processName)s |"
            "%(levelname)s: %(message)s"
        )
    )
    main()


import warnings
from dataclasses import dataclass
from typing import List, Optional

import keopscore
import torch
from pykeops.torch import Genred

from falkon.mmv_ops.utils import _get_gpu_info, _start_wait_processes, create_output_mat
from falkon.options import BaseOptions, FalkonOptions
from falkon.utils import decide_cuda
from falkon.utils.helpers import calc_gpu_block_sizes, sizeof_dtype
from falkon.utils.stream_utils import sync_current_stream

@dataclass(frozen=True)
class ArgsFmmv:
    X1: torch.Tensor
    X2: torch.Tensor
    v: torch.Tensor
    other_vars: List[torch.Tensor]
    out: torch.Tensor
    gpu_ram: float
    backend: str
    function: callable

def _decide_backend(opt: BaseOptions, num_dim: int) -> str:
    """Switch between CPU and GPU backend for KeOps"""
    if not decide_cuda(opt):
        return "CPU"
    else:
        return "GPU_1D"

def _estimate_split(N, M, D, T, R, ds):
    """Estimate the splits along dimensions N and M for a MVM to fit in memory

    The operations consist of computing the product between a kernel
    matrix (from a N*D and a M*D matrix) and a 'vector' of shape M*T
    This typically requires storage of the input and output matrices,
    which occupies (M + N)*(D + T) memory locations plus some intermediate
    buffers to perform computations.
    TODO: It is not clear how much intermediate memory KeOps requires;
    the only thing that is certain is that it is quadratic in D.
    For now we sidestep this issue by using a smaller R than what is
    actually available in GPU memory.

    This function calculates the split along N and M into blocks of size n*m
    so that we can compute the kernel-vector product between such blocks
    and still fit in GPU memory.

    Parameters
    -----------
     - N : int
        The first dimension of the kernel matrix
     - M : int
        The second dimension of the kernel matrix
     - D : int
        The data dimensionality
     - T : int
        The number of output columns
     - R : float
        The amount of memory available (in bytes)
     - ds : int
        The size in bytes of each element in the data matrices
        (e.g. 4 if the data is in single precision).

    Returns
    --------
     - n : int
        The block size to be used along the first dimension
     - m : int
        The block size along the second dimension of the kernel
        matrix

    Raises
    -------
    RuntimeError
        If the available memory `R` is insufficient to store even the smallest
        possible input matrices. This may happen if `D` is very large since we
        do not perform any splitting along `D`.

    Notes
    ------
    We find 'good' values of M, N such that
    N*(D+T) + M*(D+T) <= R/ds
    """
    R = R / ds

    # We have a linear equation in two variables (N, M)
    slope = -1
    intercept = R / (D + T)

    slack_points = 10
    # We try to pick a point at the edges such that only one kind of split
    # is necessary
    if N < intercept - 1:
        M = min(M, intercept + slope * N)
    elif M < intercept - 1:
        N = min(N, intercept + slope * M)
    else:
        # All points on the slope such that N, M > 0 are possible
        N = intercept - slack_points - 1
        M = intercept + slope * N

    if N <= 0 or M <= 0:
        raise RuntimeError("Insufficient available GPU memory (available %.2fGB)" % (R * ds / 2**30))

    return int(N), int(M)

def _single_gpu_method(proc_idx, queue, device_id):
    a: ArgsFmmv = queue.get()
    backend = a.backend
    X1 = a.X1
    X2 = a.X2
    v = a.v
    oout = a.out
    other_vars = a.other_vars
    fn = a.function
    R = a.gpu_ram

    N, D = X1.shape
    M = X2.shape[0]
    T = v.shape[1]
    device = torch.device(f"cuda:{device_id}")

    # Second round of subdivision (only if necessary due to RAM constraints)
    n, m = _estimate_split(N, M, D, T, R, sizeof_dtype(X1.dtype))

    other_vars_dev = [ov.to(device, copy=False) for ov in other_vars]
    out_ic = oout.device.index == device_id

    # Process the two rounds of splitting with a nested loop.
    with torch.cuda.device(device_id), torch.autograd.inference_mode():
        for mi in range(0, M, m):
            ml = min(m, M - mi)
            if ml != M and mi > 0:  # Then we must create a temporary output array
                out = torch.empty_like(oout)
            else:
                out = oout

            cX2 = X2[mi : mi + ml, :].to(device, copy=False)
            cv = v[mi : mi + ml, :].to(device, copy=False)

            for ni in range(0, N, n):
                nl = min(n, N - ni)
                cX1 = X1[ni : ni + nl, :].to(device, copy=False)
                cout = out[ni : ni + nl, :].to(device, copy=False)

                variables = [cX1, cX2, cv] + other_vars_dev
                fn(*variables, out=cout, device_id=device_id, backend=backend)
                if not out_ic:
                    out[ni : ni + nl, :].copy_(cout)
            if ml != M and mi > 0:
                oout.add_(out)

    return oout

def run_keops_mmv(
    X1: torch.Tensor,
    X2: torch.Tensor,
    v: torch.Tensor,
    other_vars: List[torch.Tensor],
    out: Optional[torch.Tensor],
    formula: str,
    aliases: List[str],
    axis: int,
    reduction: str = "Sum",
    opt: Optional[FalkonOptions] = None,
) -> torch.Tensor:
    if opt is None:
        opt = FalkonOptions()
    # Choose backend
    N, D = X1.shape
    T = v.shape[1]
    backend = _decide_backend(opt, D)
    data_devs = [X1.device, X2.device, v.device]

    if any(ddev.type == "cuda" for ddev in data_devs) and (not backend.startswith("GPU")):
        warnings.warn(
            "KeOps backend was chosen to be CPU, but GPU input tensors found. "
            "Defaulting to 'GPU_1D' backend. To force usage of the CPU backend, "
            "please pass CPU tensors; to avoid this warning if the GPU backend is "
            "desired, check your options (i.e. set 'use_cpu=False')."
        )
        backend = "GPU_1D"

    differentiable = any([X1.requires_grad, X2.requires_grad, v.requires_grad] + [o.requires_grad for o in other_vars])

    comp_dev_type = backend[:3].lower().replace("gpu", "cuda")  # 'cpu' or 'cuda'
    keopscore.config.config.use_cuda = comp_dev_type == "cuda"  # workaround for keops issue#248
    out = create_output_mat(
        out,
        data_devs,
        is_sparse=False,
        shape=(N, T),
        dtype=X1.dtype,
        comp_dev_type=comp_dev_type,
        other_mat=X1,
        output_stride="C",
    )
    rec_multVar_highdim = None
    if D > 100:
        rec_multVar_highdim = 1
    fn = Genred(
        formula,
        aliases,
        reduction_op=reduction,
        axis=axis,
        dtype_acc=opt.keops_acc_dtype,
        sum_scheme=opt.keops_sum_scheme,
        rec_multVar_highdim=rec_multVar_highdim,
    )
    if differentiable:
        # For differentiable inputs we don't split, since we don't know how to
        # split the backward pass.
        out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    elif comp_dev_type == "cpu" and all(ddev.type == "cpu" for ddev in data_devs):  # incore CPU
        out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    elif comp_dev_type == "cuda" and all(ddev.type == "cuda" for ddev in data_devs):  # incore CUDA
        device = data_devs[0]
        with torch.cuda.device(device):
            sync_current_stream(device)
            out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    else:  # cpu data, gpu computations: out-of-core
        # slack should be high due to imprecise memory usage estimates for keops
        gpu_info = _get_gpu_info(opt, slack=opt.keops_memory_slack)
        block_sizes = calc_gpu_block_sizes(gpu_info, N)

        args = []  # Arguments passed to each subprocess
        for i, g in enumerate(gpu_info):
            # First round of subdivision
            bwidth = block_sizes[i + 1] - block_sizes[i]
            if bwidth <= 0:
                continue
            args.append(
                (
                    ArgsFmmv(
                        X1=X1.narrow(0, block_sizes[i], bwidth),
                        X2=X2,
                        v=v,
                        out=out.narrow(0, block_sizes[i], bwidth),
                        other_vars=other_vars,
                        function=fn,
                        backend=backend,
                        gpu_ram=g.usable_memory,
                    ),
                    g.Id,
                )
            )
        _start_wait_processes(_single_gpu_method, args)

    return out


from typing import Optional, Tuple, Union

import torch.nn as nn
from diffusers.models import UNet2DConditionModel
from diffusers.models.unet_2d_blocks import UNetMidBlock2DCrossAttn
from diffusers.models.embeddings import Timesteps, TimestepEmbedding
from diffusers.configuration_utils import register_to_config

from blocks import get_down_block, get_up_block

class VideoLDM(UNet2DConditionModel):
    @register_to_config
    def __init__(
        self,
        sample_size: Optional[int] = None,
        in_channels: int = 4,
        out_channels: int = 4,
        center_input_sample: bool = False,
        flip_sin_to_cos: bool = True,
        freq_shift: int = 0,
        down_block_types: Tuple[str] = (
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "DownBlock2D",
        ),
        mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",
        up_block_types: Tuple[str] = (
            "UpBlock2D",
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
        ),
        only_cross_attention: Union[bool, Tuple[bool]] = False,
        block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
        layers_per_block: Union[int, Tuple[int]] = 2,
        downsample_padding: int = 1,
        mid_block_scale_factor: float = 1,
        act_fn: str = "silu",
        norm_num_groups: Optional[int] = 32,
        norm_eps: float = 1e-5,
        cross_attention_dim: Union[int, Tuple[int]] = 1280,
        encoder_hid_dim: Optional[int] = None,
        attention_head_dim: Union[int, Tuple[int]] = 8,
        dual_cross_attention: bool = False,
        use_linear_projection: bool = False,
        class_embed_type: Optional[str] = None,
        addition_embed_type: Optional[str] = None,
        num_class_embeds: Optional[int] = None,
        upcast_attention: bool = False,
        resnet_time_scale_shift: str = "default",
        resnet_skip_time_act: bool = False,
        resnet_out_scale_factor: int = 1.0,
        time_embedding_type: str = "positional",
        time_embedding_dim: Optional[int] = None,
        time_embedding_act_fn: Optional[str] = None,
        timestep_post_act: Optional[str] = None,
        time_cond_proj_dim: Optional[int] = None,
        conv_in_kernel: int = 3,
        conv_out_kernel: int = 3,
        projection_class_embeddings_input_dim: Optional[int] = None,
        class_embeddings_concat: bool = False,
        mid_block_only_cross_attention: Optional[bool] = None,
        cross_attention_norm: Optional[str] = None,
        addition_embed_type_num_heads=64,
    ):
        super().__init__()

        self.sample_size = sample_size

        # Check inputs
        if len(down_block_types) != len(up_block_types):
            raise ValueError(
                f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."
            )

        if len(block_out_channels) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}."
            )

        if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}."
            )

        # input
        conv_in_padding = (conv_in_kernel - 1) // 2
        self.conv_in = nn.Conv2d(
            in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding
        )

        # time
        if time_embedding_type == "fourier":
            time_embed_dim = time_embedding_dim or block_out_channels[0] * 2
            if time_embed_dim % 2 != 0:
                raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.")
            self.time_proj = GaussianFourierProjection(
                time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos
            )
            timestep_input_dim = time_embed_dim
        elif time_embedding_type == "positional":
            time_embed_dim = time_embedding_dim or block_out_channels[0] * 4

            self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
            timestep_input_dim = block_out_channels[0]
        else:
            raise ValueError(
                f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`."
            )

        self.time_embedding = TimestepEmbedding(
            timestep_input_dim,
            time_embed_dim,
            act_fn=act_fn,
            post_act_fn=timestep_post_act,
            cond_proj_dim=time_cond_proj_dim,
        )

        if encoder_hid_dim is not None:
            self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
        else:
            self.encoder_hid_proj = None

        # class embedding
        if class_embed_type is None and num_class_embeds is not None:
            self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
        elif class_embed_type == "timestep":
            self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn)
        elif class_embed_type == "identity":
            self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
        elif class_embed_type == "projection":
            if projection_class_embeddings_input_dim is None:
                raise ValueError(
                    "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
                )
            # The projection `class_embed_type` is the same as the timestep `class_embed_type` except
            # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
            # 2. it projects from an arbitrary input dimension.
            #
            # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
            # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
            # As a result, `TimestepEmbedding` can be passed arbitrary vectors.
            self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
        elif class_embed_type == "simple_projection":
            if projection_class_embeddings_input_dim is None:
                raise ValueError(
                    "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set"
                )
            self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim)
        else:
            self.class_embedding = None

        if addition_embed_type == "text":
            if encoder_hid_dim is not None:
                text_time_embedding_from_dim = encoder_hid_dim
            else:
                text_time_embedding_from_dim = cross_attention_dim

            self.add_embedding = TextTimeEmbedding(
                text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads
            )
        elif addition_embed_type is not None:
            raise ValueError(f"addition_embed_type: {addition_embed_type} must be None or 'text'.")

        if time_embedding_act_fn is None:
            self.time_embed_act = None
        elif time_embedding_act_fn == "swish":
            self.time_embed_act = lambda x: F.silu(x)
        elif time_embedding_act_fn == "mish":
            self.time_embed_act = nn.Mish()
        elif time_embedding_act_fn == "silu":
            self.time_embed_act = nn.SiLU()
        elif time_embedding_act_fn == "gelu":
            self.time_embed_act = nn.GELU()
        else:
            raise ValueError(f"Unsupported activation function: {time_embedding_act_fn}")

        self.down_blocks = nn.ModuleList([])
        self.up_blocks = nn.ModuleList([])

        if isinstance(only_cross_attention, bool):
            if mid_block_only_cross_attention is None:
                mid_block_only_cross_attention = only_cross_attention

            only_cross_attention = [only_cross_attention] * len(down_block_types)

        if mid_block_only_cross_attention is None:
            mid_block_only_cross_attention = False

        if isinstance(attention_head_dim, int):
            attention_head_dim = (attention_head_dim,) * len(down_block_types)

        if isinstance(cross_attention_dim, int):
            cross_attention_dim = (cross_attention_dim,) * len(down_block_types)

        if isinstance(layers_per_block, int):
            layers_per_block = [layers_per_block] * len(down_block_types)

        if class_embeddings_concat:
            # The time embeddings are concatenated with the class embeddings. The dimension of the
            # time embeddings passed to the down, middle, and up blocks is twice the dimension of the
            # regular time embeddings
            blocks_time_embed_dim = time_embed_dim * 2
        else:
            blocks_time_embed_dim = time_embed_dim

        # down
        output_channel = block_out_channels[0]
        for i, down_block_type in enumerate(down_block_types):
            input_channel = output_channel
            output_channel = block_out_channels[i]
            is_final_block = i == len(block_out_channels) - 1

            down_block = get_down_block(
                down_block_type,
                num_layers=layers_per_block[i],
                in_channels=input_channel,
                out_channels=output_channel,
                temb_channels=blocks_time_embed_dim,
                add_downsample=not is_final_block,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                resnet_groups=norm_num_groups,
                cross_attention_dim=cross_attention_dim[i],
                attn_num_head_channels=attention_head_dim[i],
                downsample_padding=downsample_padding,
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                only_cross_attention=only_cross_attention[i],
                upcast_attention=upcast_attention,
                resnet_time_scale_shift=resnet_time_scale_shift,
                resnet_skip_time_act=resnet_skip_time_act,
                resnet_out_scale_factor=resnet_out_scale_factor,
                cross_attention_norm=cross_attention_norm,
            )
            self.down_blocks.append(down_block)

        # mid
        if mid_block_type == "UNetMidBlock2DCrossAttn":
            self.mid_block = UNetMidBlock2DCrossAttn(
                in_channels=block_out_channels[-1],
                temb_channels=blocks_time_embed_dim,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                output_scale_factor=mid_block_scale_factor,
                resnet_time_scale_shift=resnet_time_scale_shift,
                cross_attention_dim=cross_attention_dim[-1],
                attn_num_head_channels=attention_head_dim[-1],
                resnet_groups=norm_num_groups,
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                upcast_attention=upcast_attention,
            )
        elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn":
            self.mid_block = UNetMidBlock2DSimpleCrossAttn(
                in_channels=block_out_channels[-1],
                temb_channels=blocks_time_embed_dim,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                output_scale_factor=mid_block_scale_factor,
                cross_attention_dim=cross_attention_dim[-1],
                attn_num_head_channels=attention_head_dim[-1],
                resnet_groups=norm_num_groups,
                resnet_time_scale_shift=resnet_time_scale_shift,
                skip_time_act=resnet_skip_time_act,
                only_cross_attention=mid_block_only_cross_attention,
                cross_attention_norm=cross_attention_norm,
            )
        elif mid_block_type is None:
            self.mid_block = None
        else:
            raise ValueError(f"unknown mid_block_type : {mid_block_type}")

        # count how many layers upsample the images
        self.num_upsamplers = 0

        # up
        reversed_block_out_channels = list(reversed(block_out_channels))
        reversed_attention_head_dim = list(reversed(attention_head_dim))
        reversed_layers_per_block = list(reversed(layers_per_block))
        reversed_cross_attention_dim = list(reversed(cross_attention_dim))
        only_cross_attention = list(reversed(only_cross_attention))

        output_channel = reversed_block_out_channels[0]
        for i, up_block_type in enumerate(up_block_types):
            is_final_block = i == len(block_out_channels) - 1

            prev_output_channel = output_channel
            output_channel = reversed_block_out_channels[i]
            input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]

            # add upsample block for all BUT final layer
            if not is_final_block:
                add_upsample = True
                self.num_upsamplers += 1
            else:
                add_upsample = False

            up_block = get_up_block(
                up_block_type,
                num_layers=reversed_layers_per_block[i] + 1,
                in_channels=input_channel,
                out_channels=output_channel,
                prev_output_channel=prev_output_channel,
                temb_channels=blocks_time_embed_dim,
                add_upsample=add_upsample,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                resnet_groups=norm_num_groups,
                cross_attention_dim=reversed_cross_attention_dim[i],
                attn_num_head_channels=reversed_attention_head_dim[i],
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                only_cross_attention=only_cross_attention[i],
                upcast_attention=upcast_attention,
                resnet_time_scale_shift=resnet_time_scale_shift,
                resnet_skip_time_act=resnet_skip_time_act,
                resnet_out_scale_factor=resnet_out_scale_factor,
                cross_attention_norm=cross_attention_norm,
            )
            self.up_blocks.append(up_block)
            prev_output_channel = output_channel

        # out
        if norm_num_groups is not None:
            self.conv_norm_out = nn.GroupNorm(
                num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps
            )

            if act_fn == "swish":
                self.conv_act = lambda x: F.silu(x)
            elif act_fn == "mish":
                self.conv_act = nn.Mish()
            elif act_fn == "silu":
                self.conv_act = nn.SiLU()
            elif act_fn == "gelu":
                self.conv_act = nn.GELU()
            else:
                raise ValueError(f"Unsupported activation function: {act_fn}")

        else:
            self.conv_norm_out = None
            self.conv_act = None

        conv_out_padding = (conv_out_kernel - 1) // 2
        self.conv_out = nn.Conv2d(
            block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding
        )


from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.http import MediaFileUpload
import subprocess
import os
from os.path import join

path = os.getcwd()

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/drive']

def main():
    """Shows basic usage of the Drive v3 API.
    Prints the names and ids of the first 10 files the user has access to.
    """
    # Backup the tweets
    subprocess.call(['tar -czvf tweet.tar.gz /usr/local/airflow/data/', '-1'], shell=True)

    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    print (join(path,'dags/daglibs/token.pickle'))
    if os.path.exists(join(path,'dags/daglibs/token.pickle')):
        with open(join(path,'dags/daglibs/token.pickle'), 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(join(path,
                'dags/daglibs/credentials.json'), SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open(join(path,'dags/daglibs/token.pickle'), 'wb') as token:
            pickle.dump(creds, token)

    service = build('drive', 'v3', credentials=creds)

    # Call the Drive v3 API
    file_metadata = {'name': 'tweet.tar.gz'}
    media = MediaFileUpload('/usr/local/airflow/tweet.tar.gz', mimetype='*/*')
    file = service.files().create(body=file_metadata,
                                        media_body=media,
                                        fields='id').execute()
    print ("File ID: {}".format(file.get('id')))
    if file.get('id'):
        return True
    return False

if __name__ == '__main__':
    main()


#import mxnet.ndarray as nd
from mxnet import nd
from mxnet import autograd

# REF [site] >> https://gluon-crash-course.mxnet.io/ndarray.html
def ndarray_example():
	a = nd.array(((1, 2, 3), (5, 6, 7)))
	b = nd.full((2, 3), 2.0)
	b.shape, b.size, b.dtype

	# Operations.
	x = nd.ones((2, 3))
	y = nd.random.uniform(-1, 1, (2, 3))
	x * y
	y.exp()
	nd.dot(x, y.T)

	# Indexing.
	y[1, 2]
	y[:, 1:3]
	y[:, 1:3] = 2
	y[1:2, 0:2] = 4

	# Converting between MXNet NDArray and NumPy.
	na = x.asnumpy()
	nd.array(na)

# REF [site] >> https://gluon-crash-course.mxnet.io/autograd.html
def autograd_example():
	# When differentiating a function f(x)=2x2 with respect to parameter x.
	x = nd.array([[1, 2], [3, 4]])
	x.attach_grad()

	# To let MXNet store y, so that we can compute gradients later, we need to put the definition inside a autograd.record() scope.
	with autograd.record():
		y = 2 * x * x

	# Invoke back propagation (backprop).
	#	When y has more than one entry, y.backward() is equivalent to y.sum().backward().
	y.backward()

	print('x.grad =', x.grad)

	# Using Python control flows.
	def f(a):
		b = a * 2
		while b.norm().asscalar() < 1000:
			b = b * 2
		if b.sum().asscalar() >= 0:
			c = b[0]
		else:
			c = b[1]
		return c

	a = nd.random.uniform(shape=2)
	a.attach_grad()

	with autograd.record():
		c = f(a)
	c.backward()

def main():
	ndarray_example()
	autograd_example()

#---------------------------------------------------------------------

if '__main__' == __name__:
	main()


# -*- coding: utf-8 -*-
#!/usr/bin/env python3.5

from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import RegPeopleForm, RegUserForm

def createuser(request):
    if request.method == "POST":
        uform = RegUserForm(data=request.POST)
        pform = RegPeopleForm(data=request.POST)
        if uform.is_valid() and pform.is_valid():
            user = uform.save()
            people = pform.save(commit=False)
            people = user
            people.save()
        return HttpResponseRedirect('/')
    else:
        uform=RegUserForm()
        pform=RegPeopleForm()
        return render(request, 'registration/registration.html', {'uform': uform, 'pform': pform})


from setuptools import find_packages, setup

VERSION = "0.1"

INSTALL_REQUIRES = [
    "alembic==1.9.4",
    "apischema==0.15.6",
    "asyncio==3.4.3",
    "configparser==5.3.0",
    "fastapi[all]==0.92.0",
    "psycopg2==2.9.1",
    "python-binance==1.0.16",
    "python-telegram-bot==20.0a2",
    "SQLAlchemy==1.4.37",
]

setup(
    name="report-calculation",
    version=VERSION,
    python_requires=">=3.9.0",
    packages=find_packages(exclude=["tests"]),
    author="Daniel Ducuara",
    author_email="daniel14015@gmail.com",
    description="Get a report of my porfolio",
    include_package_data=True,
    entry_points={
        "console_scripts": [
            "report-calculation = report_calculation.main:main",
            # "console = report_calulation.main:console",
        ]
    },
    install_requires=INSTALL_REQUIRES,
    extras_require={
        "dev": [
            "alembic==1.9.4",
            "bandit==1.7.0",
            "mypy==0.931",
            "pre-commit==3.1.0",
            "pylint==2.7.0",
            "black==22.10.0",
            "isort==5.10.1",
            "beautysh==6.2.1",
            "autoflake==1.7.7",
        ],
        "test": [
            "pytest==6.2.4",
            "pytest-mock==3.6.1",
            "pytest-cov==2.12.1",
            "pytest-asyncio==0.15.1",
        ],
    },
)


"""
Honk Settings.
"""

import environ
from pathlib import Path

from google.oauth2 import service_account

env = environ.Env(
    # set casting, default value
    DEBUG=(bool, False),
)

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR: Path = Path(__file__).resolve().parent.parent

# reading .env files
environ.Env.read_env(BASE_DIR / '.env')

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY: str = env('SECRET_KEY')

DEBUG: bool = env('DEBUG')

ALLOWED_HOSTS: list[str] = ['localhost', '127.0.0.1', 'honk.rafaelmc.net']

CSRF_TRUSTED_ORIGINS: list[str] = ['https://honk.rafaelmc.net']

# Application definition

INSTALLED_APPS: list[str] = [
    'circus.apps.CircusConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_extensions',
    'rest_framework',
    'rest_framework.authtoken',
]

MIDDLEWARE: list[str] = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'honk.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [BASE_DIR / "templates"],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'honk.wsgi.application'

# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': BASE_DIR / 'sqlite_data' / 'db.sqlite3',
    }
}

# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators

VALIDATOR_PATH = 'django.contrib.auth.password_validation.'

AUTH_PASSWORD_VALIDATORS = [
    {'NAME': VALIDATOR_PATH + 'UserAttributeSimilarityValidator'},
    {'NAME': VALIDATOR_PATH + 'MinimumLengthValidator'},
    {'NAME': VALIDATOR_PATH + 'CommonPasswordValidator'},
    {'NAME': VALIDATOR_PATH + 'NumericPasswordValidator'},
]

# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.2/howto/static-files/

STATIC_URL = 'static/'

STATICFILES_DIRS = [
    BASE_DIR / "static",
]

# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
LOGIN_REDIRECT_URL = "/"
STORAGES = {
    "default": {"BACKEND": "storages.backends.gcloud.GoogleCloudStorage"},
    "staticfiles": {
        "BACKEND": "storages.backends.gcloud.GoogleCloudStorage"
    },
}

# GOOGLE_APPLICATION_CREDENTIALS =
GS_BUCKET_NAME = 'honkhonk'
MEDIA_URL = "/media/"
MEDIA_ROOT = BASE_DIR / "media"
GS_CREDENTIALS = service_account.Credentials.from_service_account_file(
    f"{BASE_DIR}/gcp-honk-credentials.json"
)

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': [
        'rest_framework.authentication.TokenAuthentication',
    ],
}


import collections

from oneview_redfish_toolkit.api.errors import \
    OneViewRedfishException
from oneview_redfish_toolkit.api.errors import \
    OneViewRedfishResourceNotFoundException
from oneview_redfish_toolkit.api.redfish_json_validator import \
    RedfishJsonValidator
from oneview_redfish_toolkit import config

class RedfishError(RedfishJsonValidator):
    """Creates a Redfish Error Dict

        Populates self.redfish with errors. Will not validate as there's no
        schema to validate against.

    """

    SCHEMA_NAME = None

    def __init__(self, code, message):
        """Constructor

            Populates self.redfish with error message.
        """

        super().__init__(self.SCHEMA_NAME)
        self.redfish["error"] = collections.OrderedDict()
        # Check if Code is a valid Code Error in the registry
        if code not in config.get_registry_dict()["Base"]["Messages"]:
            raise OneViewRedfishResourceNotFoundException(
                "Registry {} not found.".format(code)
            )

        self.redfish["error"]["code"] = "Base.1.1." + code
        self.redfish["error"]["message"] = message
        self.redfish["error"]["@Message.ExtendedInfo"] = list()

    def add_extended_info(
        self,
        message_id,
        message_args=[],
        related_properties=[]):
        """Adds an item to ExtendedInfo list using values from DMTF registry

            Adds an item to ExtendedInfo list using the values for Message,
            Severity and Resolution from DMTF Base Registry.

            Parameters:
                message_id: Id of the message; oneOf the keys in Redfish
                    Registry Messages
                message_args: List of string to replace markers on Redfish
                    messages. Must have the same length as the number of %
                    signs found in the registry Message field
                related_properties: Properties relates to this e error if
                    necessary

        """
        messages = config.get_registry_dict()["Base"]["Messages"]

        # Verify if message_id exists in registry
        try:
            severity = messages[message_id]["Severity"]
        except Exception:
            raise OneViewRedfishResourceNotFoundException(
                "Message id {} not found.".format(message_id)
            )

        message = messages[message_id]["Message"]

        # Check if numbers of replacements and message_args length match
        replaces = message.count('%')
        replacements = len(message_args)
        if replaces != replacements:
            raise OneViewRedfishException(
                'Message has {} replacements to be made but {} args '
                'where sent'.format(replaces, replacements)
            )

        # Replacing the marks in the message. A better way to do this
        # is welcome.
        for i in range(replaces):
            message = message.replace('%' + str(i + 1), message_args[i])

        # Construct the dict
        extended_info = collections.OrderedDict()
        extended_info["@odata.type"] = "#Message.v1_0_5.Message"
        extended_info["MessageId"] = "Base.1.1." + message_id
        extended_info["Message"] = message
        extended_info["RelatedProperties"] = related_properties
        extended_info["MessageArgs"] = message_args
        extended_info["Severity"] = severity
        extended_info["Resolution"] = messages[message_id]["Resolution"]

        # Append it to the list
        self.redfish["error"]["@Message.ExtendedInfo"].append(extended_info)


from dataclasses import dataclass, asdict, field, make_dataclass
from typing import List, Union, Any, Dict
from enum import Enum
from sigrok.core.classes import ConfigKey

__all__ = [
    "SrBlankState",
    "SrDeviceState",
    #"SrFileState",
    #"AnalogChannel",
    #"LogicChannel",
]

colorsArray = [
    '#fce94f', '#fcaf3e', '#e9b96e', '#8ae234', '#729fcf', '#ad7fa8', '#cf72c3', '#ef2929',
    '#edd400', '#f57900', '#c17d11', '#73d216', '#3465a4', '#75507b', '#a33496', '#cc0000',
    '#c4a000', '#ce5c00', '#8f5902', '#4e9a06', '#204a87', '#5c3566', '#87207a', '#a40000',
    '#16191a', '#2e3436', '#555753', '#888a8f', '#babdb6', '#d3d7cf', '#eeeeec', '#ffffff'
]

def factory(data):
    return dict(x for x in data if x[1] is not None)

def build_opts(opts: list):
    #NOTE: key, id, name, desc, keyName, caps
    data = []
    for opt in opts:
        bases = []
        opt_fields = []
        if 'caps' in opt:
            if 'LIST' in opt['caps']:
                bases.append(ConfList)
            elif 'GET' in opt['caps']:
                bases.append(ConfValue)

        for k, v in opt.items():
            opt_fields.append((k, type(v)))

        opt_fields.append( ('keyName', str))

        keyName=ConfigKey.get(opt['key']).name

        SrOpt = make_dataclass(cls_name='SrOpt', fields=opt_fields, bases=tuple(bases))
        srOpt = SrOpt(**opt, keyName = keyName)

        data.append( (keyName, srOpt) )

    return data

@dataclass
class ConfValue:
    value: Any

@dataclass
class ConfList:
    values: List[Any] = field(default_factory=list)

@dataclass
class Channel:
    name: str
    text: str
    color: str
    enabled: bool
    index: int
    type: str

    def update(self, opts: Dict):
        for key, value in opts.items():
            if hasattr(self, key):
                setattr(self, key, value)

@dataclass
class LogicChannel(Channel):
    traceHeight: int = 34

@dataclass
class AnalogChannel(Channel):
    pVertDivs: int = 1
    nVertDivs: int = 1
    divHeight: int = 51
    vRes: float = 20.0
    autoranging: bool = True
    conversion: str = ''
    convThres: str = ''
    showTraces: str = ''

class ChTypesEnum(Enum):
    analog = AnalogChannel
    logic = LogicChannel

@dataclass
class ChnlsContBase:
    def set(self, chOpts: List[Dict] ):
        result = []
        for opts in chOpts:
            chName = opts.pop('chName', None)
            if hasattr(self, chName):
                attr = getattr(self, chName)
                attr.update(opts)
                result.append(chName)
        return result

    def get(self):
        data = asdict(self, dict_factory=factory)
        return data.values()

    def get_list(self):
        data = asdict(self, dict_factory=factory)
        return list( { item['type'] for item in data.values() })

@dataclass
class OptsContBase:
    def set(self, opts: List[Dict] ):
        result = []
        for item in opts:
            optName = item['keyName']
            if hasattr(self, optName):
                attr = getattr(self, optName)
                attr.value = item['value']
                result.append(optName)
        return result

    def get(self):
        return asdict(self, dict_factory=factory)

    def get_list(self):
        data = [ item['key'] for item in asdict(self).values() ] #list(asdict(self).values()
        return  data

def make_ch_container(channels: List[Dict]):
    fields = []
    for i, item in enumerate(channels):
        chInst = ChTypesEnum[item['type']].value(**item, color= colorsArray[i], text=item['name'])
        fields.append( ( item['name'], chInst) )

    ChnlsContainer = make_dataclass('ChnlsContainer', [ item[0] for item in fields], bases=tuple([ChnlsContBase]) )
    data = ChnlsContainer(**{ item[0]: item[1] for item in fields})
    return data

def make_opts_container(opts: List[Dict]):
    fields = build_opts(opts)
    OptsContainer = make_dataclass('OptsContainer', [ item[0] for item in fields], bases=tuple([OptsContBase]))
    data = OptsContainer(**{ item[0]: item[1] for item in fields})
    return data

#-----------------  STATE TYPES   -----------------#
@dataclass
class SrBlankState:
    id: str
    name: str
    sourcename: str
    type: str = field(init=False)

    def __post_init__(self):
        self.type = "BLANK"

    def get(self):
        data = asdict(self, dict_factory=factory)
        return data

    def copy(self):
        return self.__class__(self.get())

@dataclass
class SrDeviceState(SrBlankState):
    drvopts: Dict[ str, Any ]
    devopts: Dict[ str, Any ]
    channels: Dict[ str, Union[ AnalogChannel, LogicChannel ] ]#Any

    def __init__(self, id, name, sourcename, drvopts: List, devopts: List, channels: List[ChTypesEnum]):
        SrBlankState.__init__(self, id, name, sourcename)
        self.type = 'DEVICE'
        self.drvopts = make_opts_container(drvopts)
        self.devopts = make_opts_container(devopts)

        self.channels = make_ch_container(channels)

    def get(self):
        data = super().get()

        data['drvopts'] = self.drvopts.get_list()
        data['devopts'] = self.devopts.get_list()
        data['channels'] = self.channels.get_list()

        return data


from django.shortcuts import render, redirect
from app14.models import ProductModel

def openmainpage(request):
    if request.method == "POST":
        name = request.POST.get("product_name")
        price = request.POST.get("product_price")
        photo = request.FILES["product_photo"]

        ProductModel(name=name, price=price, photo=photo).save()
        return redirect('main')
    else:
        return render(request, 'main.html')


import torch
import numpy as np
from shapmagn.global_variable import DATASET_POOL
from shapmagn.utils.obj_factory import partial_obj_factory

# todo reformat the import style
class DataManager(object):
    def __init__(
        self,
    ):
        """
        the class for data management
        return a dict, each train/val/test/debug phase has its own dataloader
        """
        self.data_path = None
        self.data_opt = None
        self.phases = []

    def set_data_path(self, data_path):
        """
        set data path
        :param data_path:
        :return:
        """
        self.data_path = data_path

    def set_data_opt(self, data_opt):
        """
        set data opt
        :param data_opt:
        :return:
        """
        self.data_opt = data_opt

    def init_dataset_loader(self, transformed_dataset, batch_size):
        """
        initialize the data loaders: set work number, set work type( shuffle for trainning, order for others)
        :param transformed_dataset:
        :param batch_size: the batch size of each iteration
        :return: dict of dataloaders for train|val|test|debug
        """

        def _init_fn(worker_id):
            np.random.seed(12 + worker_id)

        num_workers_reg = {
            "train": 8,
            "val": 8,
            "test": 8,
            "debug": 4,
        }  # {'train':0,'val':0,'test':0,'debug':0}#{'train':8,'val':4,'test':4,'debug':4}
        shuffle_list = {"train": True, "val": False, "test": False, "debug": False}
        batch_size = (
            [batch_size] * 4 if not isinstance(batch_size, list) else batch_size
        )
        batch_size = {
            "train": batch_size[0],
            "val": batch_size[1],
            "test": batch_size[2],
            "debug": batch_size[3],
        }
        dataloaders = {
            x: torch.utils.data.DataLoader(
                transformed_dataset[x],
                batch_size=batch_size[x],
                shuffle=shuffle_list[x],
                num_workers=num_workers_reg[x],
                worker_init_fn=_init_fn,
                pin_memory=True,
            )
            for x in self.phases
        }
        return dataloaders

    def build_data_loaders(self, batch_size=20, is_train=True):
        """
        build the data_loaders for the train phase and the test phase
        :param batch_size: the batch size for each iteration
        :param is_train: in train mode or not
        :return: dict of dataloaders for train phase or the test phase
        """
        if is_train:
            self.phases = ["train", "val", "debug"]
        else:
            self.phases = ["test"]
        name = self.data_opt["name"]
        dataset_opt = self.data_opt[(name, {}, "settings for {} dataset".format(name))]
        assert name in DATASET_POOL, "{} not in dataset pool {}".format(
            name, DATASET_POOL
        )
        if name!="custom_dataset":
            transformed_dataset = {
                phase: DATASET_POOL[name](self.data_path, dataset_opt, phase=phase)
                for phase in self.phases
            }
        else:
            transformed_dataset = {
                phase: partial_obj_factory(dataset_opt["name"])(self.data_path, dataset_opt, phase=phase)
                for phase in self.phases
            }
        dataloaders = self.init_dataset_loader(transformed_dataset, batch_size)
        dataloaders["data_size"] = {
            phase: len(dataloaders[phase]) for phase in self.phases
        }
        print("dataloader is ready")

        return dataloaders


import webapp2, jinja2, os

# Import requests with app engine adapter
import requests, requests_toolbelt.adapters.appengine
from bs4 import BeautifulSoup

import datetime, time

from google.appengine.ext import ndb

# Patch adapter
requests_toolbelt.adapters.appengine.monkeypatch()

# os.path.dirname(__file__) is the current location of the file
# os.path.join joins the current location with templates
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader = jinja2.FileSystemLoader(template_dir), autoescape = True)
baseUrl = "https://woodroffehs.ocdsb.ca/Home%20Page%20Images/"
srcLocation = 'http://woodroffehs.ocdsb.ca/Home%20Page%20Images/Forms/AllItems.aspx'

garbageImages = [
    '',
]

acceptableFormats = [
    '.png',
    '.PNG',
    '.jpg',
    '.JPG'
]

for item in range(len(garbageImages)):
    garbageImages[item] = srcLocation + garbageImages[item]

def GetImages():
    images = []
    page = BeautifulSoup(requests.get(srcLocation).content)
    for image in page.findAll('img'):
        #images.append(image)
        try:
            alt = str(image['alt']).replace(" ", '%20')
            if alt not in garbageImages and any(acceptable in alt for acceptable in acceptableFormats):
                images.append(baseUrl + alt)
        except KeyError:
            pass
    return images

class Handler(webapp2.RequestHandler):
    def write(self, *a, **kw):
        self.response.out.write(*a, **kw)

    def render_str(self, template, **params):
        t = jinja_env.get_template(template)
        return t.render(params)

    def render(self, template, **kw):
        self.write(self.render_str(template, **kw))

# MainPage is a child of Handler, therefore it has all the functions and variables of Handler
class MainPage(Handler):
    '''
    def get(self):
        now = datetime.datetime.now()
        latestUpdateObj = Timeout.query().fetch()[0]
        latestUpdate = latestUpdateObj.updateTime

        images = Image.query().fetch()
        self.render('slideshow.html', images=images)
        if now >= now:
            latestUpdate.updateTime = now + datetime.timedelta(hours=2)
            ndb.delete_multi(Image.query().fetch())
            for image in GetImages():
                img = Image(link = image)
                img.put()
        #self.render('slideshow.html', images=GetImages())
    '''

    def get(self):
        now = datetime.datetime.now()

class GenerateInitial(Handler):
    def get(self):
        timeout = Timeout(updateTime = datetime.datetime.now())
        timeout.put()

        for image in GetImages():
            img = Image(link = image)
            img.put()

class EscapeCache(Handler):
    def get(self):
        self.render('slideshow.html', images=GetImages())

class Image(ndb.Model):
    link = ndb.StringProperty(required = True)
    uuid = ndb.StringProperty()
    updateTime = ndb.DateTimeProperty()

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/init', GenerateInitial),
    ('/escape', EscapeCache)
], debug=True)


from ..nmf import (PreambleEndRecord, ViaRecord, VersionRecord,
                   ModeRecord, KnownEncodingRecord, UpgradeRequestRecord,
                   UpgradeResponseRecord, Record, register_types,
                   SizedEnvelopedMessageRecord, PreambleAckRecord, EndRecord)
from .gssapi import GSSAPIStream

class NMFStream:
    def __init__(self, stream, url, server_name=None):
        self._inner = stream
        self._server_name = server_name
        self.url = url

        register_types()

    def preamble(self):
        data = [
            VersionRecord(MajorVersion=1, MinorVersion=0),
            ModeRecord(Mode=2),
            ViaRecord(ViaLength=len(self.url), Via=self.url),
            KnownEncodingRecord(Encoding=8),
        ]

        self._inner.write(b''.join(d.to_bytes() for d in data))

        if self._server_name:
            msg = UpgradeRequestRecord(UpgradeProtocolLength=21,
                                       UpgradeProtocol='application/negotiate').to_bytes()
            self._inner.write(msg)
            d = self._inner.read(1)
            if d != UpgradeResponseRecord().to_bytes():
                raise IOError('Negotiate not supported')

            self._inner = GSSAPIStream(self._inner, self._server_name)

        self._inner.write(PreambleEndRecord().to_bytes())

        if self._inner.read(1) != PreambleAckRecord().to_bytes():
            raise IOError('Preamble end not acked')

    def write(self, data):
        msg = SizedEnvelopedMessageRecord(Size=len(data), Payload=data)
        self._inner.write(msg.to_bytes())

    def read(self, count=None):
        if count:
            data = self._inner.read(count)
            s, msg = Record.parse(data)
            return data[s:msg.Size+s]
        else:
            msg = Record.parse_stream(self._inner)

            return msg.Payload

    def close(self):
        self._inner.write(EndRecord().to_bytes())
        self._inner.close()