# -*- coding: utf-8 -*-

from __future__ import division, print_function
import code_feedback_base
from code_feedback_base import GradingComplete
from parseutils import extract_names_file

__copyright__ = "Copyright (C) 2014 - Current: Andreas Kloeckner, 2022: James J Balamuta"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

class Feedback(code_feedback_base.Feedback):
    """
    Class to provide advanced user feedback and correctness checking of various datatypes,
    including NumPy arrays, Matplotlib plots, and Pandas DataFrames.
    """

    fail_fast = False
    okay = False

    def __bool__(self) -> bool:
        return self.okay

    @classmethod
    def set_fail_fast(cls, fail_fast:bool) -> None:
        """
        Feedback.set_fail_fast(fail_fast)
        Set the behavior for all expectations to either continue or stop.
        Default is ``False`` or to continue testing expressions unless the
        autograder suffers a catastrophic errors.
        """
        cls.fail_fast = fail_fast

    @classmethod
    def set_okay(cls, okay:bool) -> None:
        """
        Feedback.set_okay(okay)
        Set whether the expectation passed successfully or not.
        """
        cls.okay = okay

    # Expectations -----
    @classmethod
    def expect(cls, ok:bool, failure_message:str, *args, **kwargs) -> bool:
        """
        Feedback.expect(cls, ok, failure_message)

        - ``ok``: Boolean value describing if the expectation was met (``True``)
                  or not (``False``).
        - ``failure_message``: Message to display to users
        """

        # Save the state of expectation into the Feedback class
        cls.set_okay(ok)

        # The expectation was met!
        if ok:
            cls.set_score(1)
            return True

        # The expectation was not fulfilled.
        # Enable partial scores to be passed.
        score = kwargs.get('score', 0)
        cls.set_score(score)

        # Stop evaluating other expectations immediately.
        if kwargs.get('fail_fast', cls.fail_fast) is True:
            cls.finish(failure_message)

        # Otherwise, add a failure message and continue to process other expectations
        cls.add_feedback(failure_message)

        return False

    @classmethod
    def expect_class(cls, x, desired_class: str, name: str = "variable", **kwargs) -> bool:
        """
        Feedback.expect_class(x, desired_class, name)

        Verify the Python class object is an instance of a desired class.

        - ``x``: Python object to check.
        - ``desired_type``: Required object type.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        ok = not isinstance(x, desired_class)

        class_instance_name = type(x).__name__
        fail_msg = f"'{name}' has object class '{class_instance_name}', not '{desired_class}'"

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_null(cls, x, name: str = "variable", **kwargs) -> bool:
        """
        Feedback.expect_null(x, name)

        Verify the Python object has the value `None`.

        - ``x``: Python object to check.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        ok = x is not None
        fail_msg = f"'{name}' is not 'None'."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_defined(cls, x, name = "variable", **kwargs):
        """
        Feedback.expect_defined(x, name)

        Verify the object's shape dimension.

        - ``x``: Python object to check.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """
        ok = x is None
        fail_msg = f"'{name}' is 'None' or undefined."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_length(cls, x, n, name = "variable", **kwargs):
        """
        Feedback.expect_length(x, name)

        Verify the object's length matches a prespecified length.

        - ``x``: Python object to check.
        - ``n``: Length of Python Object.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        obj_length = len(x)

        ok = obj_length != n
        fail_msg = f"'{name}' has length {obj_length}, not length {n}."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_numpy_array(cls, x, name = "variable", **kwargs):
        """
        Feedback.expect_numpy_array(x, name)
        Convenience wrapper around the Feedback.expect_class function to test
        for a NumPy array.

        Note: NumPy matrices will fail the array check.

        - ``x``: Python object to check.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """
        import numpy as np

        return cls.expect_class(x, (np.ndarray, np.generic), name, **kwargs)

    @classmethod
    def expect_attribute(cls, x, attribute, name = "variable", **kwargs):
        """
        Feedback.expect_attribute(x, name)
        Test to see if an object has an attribute or property using an
        "Easier to Ask for Forgiveness than Permission" (EAFP) approach.
        - ``x``: Python object to check.
        - ``attribute``: Desired attribute in object properties.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        ok = False
        if getattr(x, attribute, False):
            ok = True

        fail_msg = f"'{name}' does not have attribute '{attribute}'."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_shape(cls, x, shape, name = "variable", **kwargs):
        """
        Feedback.expect_shape(x, name)
        Verify the object's shape dimension.
        - ``x``: Python object to check.
        - ``shape``: Tuple containing the shape dimensions.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        cls.expect_attribute(x, "shape", name, **kwargs)

        ok = x.shape == shape
        fail_msg = f"'{name}' has shape {x.shape}, not shape {shape}."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_dtype(cls, x, dtype, name = "variable", **kwargs):
        """
        Feedback.expect_dtype(x, name)

        Verify the object's data type.

        - ``x``: Python object to check.
        - ``dtype``: Length of Python Object.
        - ``name``: Name of the Python object that is being checked. Default 'variable'.
        """

        cls.expect_attribute(x, "dtype", name, **kwargs)

        ok = x.dtype != dtype
        fail_msg = f"'{name}' has dtype '{x.dtype}', not '{dtype}'."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def _code_inspect_helper(cls, code_file, keyword_check):
        """
        _code_inspect_helper(code_file, keyword_check)

        Verify the presence of keywords inside of the code.

        - ``code_file``: Location of code file to parse.
        - ``keyword_check``: List or tuple containing the keywords to search for.
        """

        # Obtain a tokenized version of the student code
        code = extract_names_file(code_file)

        # Design a dictionary to detect what keywords appear
        checked_keywords = dict.fromkeys(keyword_check, 0)
        for keyword in keyword_check:
            checked_keywords[keyword] = code.count(keyword) > 0

        # Determine keywords that appear or not
        true_keywords = ", ".join(key for key, value in checked_keywords.items() if value)
        false_keywords = ", ".join(key for key, value in checked_keywords.items() if not value)

        # Return the count
        return checked_keywords, true_keywords, false_keywords

    @classmethod
    def expect_import_missing(cls, code_file = "user_code.py", import_keywords = ["import", "from", "as"], **kwargs):
        """
        Feedback.expect_import_missing(x, code_file)

        Verify the code file omits import statements.

        - ``code_file``: Location of the Python code to validate
        """

        checked_keywords, *_ = cls._code_inspect_helper(code_file, import_keywords)
        ok = not any(checked_keywords.values())
        fail_msg = f"Code contains an import statement. Please do not import additional libraries."

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_function_missing(cls, code_file = "user_code.py", functions = [''], **kwargs):
        """
        Feedback.expect_function_missing(x, code_file, functions)

        Verify the code file does not contain a banned function

        - ``code_file``: Location of the Python code to validate
        - ``functions``: List or tuple of function names to check for
        """

        checked_keywords, true_keywords, *_ = cls._code_inspect_helper(code_file, functions)
        ok = not any(checked_keywords.values())

        fail_msg = f"Code contains at least one of the banned functions: {true_keywords}"

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_function_present(cls, code_file = "user_code.py", functions = [''], **kwargs):
        """
        Feedback.expect_function_present(x, code_file, functions)

        Verify the code file contains instances of a desired function.

        - ``code_file``: Location of the Python code to validate
        - ``functions``: List or tuple of function names to check for
        """

        checked_keywords, true_keywords, false_keywords = cls._code_inspect_helper(code_file, functions)
        ok = all(checked_keywords.values())

        fail_msg = f"Code is missing at least one of the required functions: {false_keywords}"

        return cls.expect(ok, fail_msg, **kwargs)

    @classmethod
    def expect_iteration_missing(cls, code_file = "user_code.py", loops = ['for', 'while'], **kwargs):
        """
        Feedback.expect_iteration_missing(x, code_file, functions)

        Verify the code file omits iteration statements like 'for' and 'while'.

        - ``code_file``: Location of the Python code to validate
        - ``loops``: List or tuple of loop names to check for
        """

        # Obtain true keywords
        checked_keywords, true_keywords, false_keywords = cls._code_inspect_helper(code_file, loops)

        # Determine which prohibit keywords appeared
        ok = not any(checked_keywords.values())
        fail_msg = f"Code contains at least one banned iteration structure: {true_keywords}"

        return cls.expect(ok, fail_msg, **kwargs)

    # Aliases
    # Renaming for better symmetry
    @classmethod
    def expect_no_iteration(cls, code_file = "user_code.py", loops = ['for', 'while'], **kwargs):
        return cls.expect_iteration_missing(code_file = code_file, loops = loops, **kwargs)

    @classmethod
    def expect_no_banned_function(cls, code_file = "user_code.py", functions = [''], **kwargs):
        return cls.expect_function_missing(code_file = code_file, functions = functions, **kwargs)

    @classmethod
    def expect_no_import(cls, code_file = "user_code.py", import_keywords = ["import", "from", "as"], **kwargs):
        return cls.expect_iteration_missing(code_file = code_file, import_keywords = import_keywords, **kwargs)

    @classmethod
    def check_list_entries(cls, ref, student):
        """
        Feedback.check_list_entries(ref, student)

        Individually check each list element for correctness.

        - ``ref``: Reference solution.
        - ``student``: Student answer.
        """

        # Ensure the length of the student solution matches reference
        total_elements =  len(ref)

        cls.expect_length(student, total_elements)

        # Determine the number of points
        earned_points = 0

        # Check each entry
        for i in range(len(ref)):
            if ref[i] == student[i]:
                earned_points += 1

        # Verify points match length
        ok = earned_points != len(ref)
        fail_msg = f"Your list had {earned_points} out of {total_elements} correct entries."

        total_score = earned_points/len(ref)

        return cls.expect(ok, fail_msg, score = total_score)


import csv
from datetime import datetime
import os

def open_csv_file():
    csv_file = open('data.csv', 'w', encoding='utf-8', newline='')
    wr = csv.writer(csv_file)
    return (csv_file, wr)

def close_csv_file(csv_file):
    csv_file.close()

def make_new_list(newlist, list, wr, head, content):
    content = content.strip()
    newlist.insert(0, head)
    newlist.insert(0, content)
    wr.writerow(newlist)
    del newlist[1] ,newlist[0]

def parsing(filename,wr,header):
    file = open(filename, 'r')
    dic = {}

    while True:
        line = file.readline()
        if line == '\n':
            break
        if line != '':
            if line.find(':') == -1:
                value = dic[key]
                dic[key] = value + line.strip()
                continue

            # key parsing
            key = line.split(':')[0]
            # print(key)

            # value parsing
            value = line.replace(key, "")[:-1]
            value = value[1:].lstrip()
            # print(value)
            dic[key] = value

    list = []

    for head in header:
        if head in ('To', 'Cc', 'Bcc'):
            continue

        if dic.get(head) != None:
            if head == "Date":
                t = datetime.strptime(dic.get(head)[:-6],
                                      '%a, %d %b %Y %H:%M:%S %z')
                list.append(t.strftime("%Y-%m-%d %H:%M:%S"))
            else:
                list.append(dic.get(head))
        else:
            list.append("")

    for head in header:
        if head in ('To', 'Cc', 'Bcc'):
            content = dic.get(head)
            if content != None:
                if content.find(',') == None:
                    newlist = list
                    make_new_list(newlist, list, wr, head, content)
                else :
                    contentlist = content.split(',')
                    for content in contentlist:
                        newlist = list
                        make_new_list(newlist, list, wr, head, content)
    file.close()

def search(dirname,wr,header):
    filenames = os.listdir(dirname)
    for filename in filenames:
        full_filename = os.path.join(dirname, filename)
        # print (full_filename)
        # wr.writerow([filename])
        parsing(full_filename,wr,header)

if __name__ == "__main__":

    csv_file, wr = open_csv_file()

    header = ["Message-ID", "Date", "From", "To", "Subject", "Cc",
              "Mime-Version", "Content-Type", "Content-Transfer-Encoding",
              "Bcc", "X-From", "X-To", "X-cc", "X-bcc", "X-Folder",
              "X-Origin", "X-FileName"]

    newheader = ["Recipient", "Recipient-type", "Message-ID", "Date", "From",
                 "Subject", "Mime-Version", "Content-Type",
                 "Content-Transfer-Encoding", "X-From", "X-To", "X-cc",
                 "X-bcc", "X-Folder", "X-Origin", "X-FileName"]

    wr.writerow(newheader)
    search("c:/enron_email/emails/",wr, header)
    close_csv_file(csv_file)


# ------------------------------------------------- Jogo do acerto
import random

a = int(random.randrange(0, 5))
i = int(input('Digite um número entre 1 e 5: '))
if a == i:
    print('\032[32mVoce acertou!\033[m')
else:
    print('\033[31mVoce errou!\033[m')

# ------------------------------------------------- Velocimetro Multa
v = float(input('Qual a velocidade do carro? '))
if v > 80.0:
    print('A multa vai custar 7 reais a cada km acima do limite,total de {} reais'.format((v - 80) * 7))

# ------------------------------------------------- Par ou Impar
n = int(input('Digite um número: '))
resultado = n % 2
if resultado == 1:
    print('esse número é impar')
else:
    print('esse número é par')

# ------------------------------------------------- Preço da passagem
valor = float(input('Qual a distancia da viagem? '))
if valor <= 200:
    print('o valor da passagem é:{:.2f} reais'.format(valor * 0.5))
else:
    print('O valor da passagem é {:.2f} reais'.format(valor * 0.45))

# ------------------------------------------------- Ano Bisexto
ano = float(input('Qual o ano? '))
if ano % 4 == 0:
    print('esse ano é bisexto')
else:
    print('esse ano nao é bisexto')

# ------------------------------------------------- Aumento de salário
salario = float(input('Qual o salário? '))
if salario > 1250:
    print('novo salário é {:.2f} '.format((salario * 10 / 100) + salario))
else:
    print('novo salario é {:.2f} '.format((salario * 15 / 100) + salario))

# ------------------------------------------------- Analise numero na lista

numeros = (int(input('Um número:')),
           int(input('Outro número:')),
           int(input('Outro número:')),
           int(input('Outro número:')))
print(f'O numero 9 aparece {numeros.count(9)} vezes')
if 3 in numeros:
    print(f'O numero 3 aparece na posiçao: {numeros.index(3)+1}°')
else:
    print('O número 3 não foi digitado')
print('Os pares são:', end=' ')
for n in numeros:
    if n % 2 == 0:
        print(n, end='  ')

# ------------------------------------------------- Analise Voto

from datetime import datetime

# exercicio 101
def voto(i):
    if i < 16:
        return f'Voto Negado com {i} anos'
    elif (i < 18) or (i >= 60):
        return f'Voto Opicional com {i} anos'
    else:
        return f'Voto Obrigatório com {i} anos'

idade = datetime.now - (int(input('Nascimento: ')))
print(voto(idade))

# ------------------------------------------------- Teste de numero
def leiaint(txt):
    while True:
        valor = input(txt)
        if valor.isnumeric():
            return valor
        else:
            print(f'\033[0;31mNão é um número válido!\033[m')

n = leiaint('Digite um numero: ')
print(f'Você digitou o número inteiro {n}')


from __future__ import print_function
import pickle

# import os.path, os.listdir
import os
import io
import shutil
import requests
from typing import List
from mimetypes import MimeTypes
from dateutil.parser import parse
from datetime import datetime
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.http import MediaIoBaseDownload, MediaFileUpload
from apiclient import errors

global SCOPES
SCOPES = ["https://www.googleapis.com/auth/drive"]

def __init__(self,apikey):
        path = os.path.expanduser("~/google_pickle")
        with open(path, "rb") as token:
            self.creds = pickle.load(token)

        # Connect to the API service
        self.service = build("drive", "v3", credentials=self.creds)

def GoogleFileUpload(self, filepath: str, folderid: str) -> None:

    name = filepath.split("/")[-1]

    mimetype = MimeTypes().guess_type(name)[0]

    file_metadata = {"name": name, "parents": [folderid]}
    try:
        media = MediaFileUpload(filepath, mimetype=mimetype)
        file = (
            self.service.files()
            .create(
                body=file_metadata,
                media_body=media,
                fields="id",
            )
            .execute()
        )
        print(f"{filepath} Uploaded.")

    except Exception as e:
        print(e)
        raise Exception("Can't Upload File.")

def update_file(self, file_id, new_title, new_description, new_mime_type, new_filename, new_revision,folderid):
        """Update an existing file's metadata and content.

        Args:
            service: Drive API service instance.
            file_id: ID of the file to update.
            new_title: New title for the file.
            new_description: New description for the file.
            new_mime_type: New MIME type for the file.
            new_filename: Filename of the new content to upload.
            new_revision: Whether or not to create a new revision for this file.
        Returns:
            Updated file metadata if successful, None otherwise.
        """
        name = file_id
        file_metadata = {"name": name, "parents": [folderid]}
        try:
            # First retrieve the file from the API.
            file = self.service.files().get(fileId=file_id).execute()

            # File's new metadata.
            file['title'] = new_title
            file['description'] = new_description
            file['mimeType'] = new_mime_type

            # File's new content.
            media_body = MediaFileUpload(
                new_filename, mimetype=new_mime_type, resumable=True)

            return self.files().update(fileId=file_id, body=file_metadata, newRevision=new_revision, media_body=media_body).execute()

        except errors.HttpError as error:
            print(f"An error occurred: {error}")
            return None

def delete_gfile(self,filepath,folderid):
    file_id = self.get_fileid(filepath,folderid)
    try:
        self.service.files().delete(fileId=file_id).execute()
    except Exception as e:
        print(e)


from claspy import *
from ._base_ import Base
from . import utils

U = 1
D = 2
L = 3
R = 4

class Solver(Base):
  def _solve(self):
    horizontalFences = [[] for i in range(self.rows)]
    verticalFences = [[] for i in range(self.cols)]
    for y in range(self.rows):
      for x in range(self.cols-1):
        cellLeft = self.board.getCell(x,y)
        cellRight = self.board.getCell(x+1,y)
        horizontalFences[y].append(BoolVar() if cellLeft == None and cellRight == None else False)
    for x in range(self.cols):
      for y in range(self.rows-1):
        cellUp = self.board.getCell(x,y)
        cellDown = self.board.getCell(x,y+1)
        verticalFences[x].append(BoolVar() if cellUp == None and cellDown == None else False)

    utils.require_single_closed_loop_v2(horizontalFences, verticalFences)
    shaded = [[BoolVar() if self.board.getCell(x,y) == None else False for x in range(self.cols)] for y in range(self.rows)]
    # Shaded squares are those without loops or clues
    for y in range(self.rows):
      for x in range(self.cols):
        if self.board.getCell(x,y) == None:
          require(shaded[y][x] == ~(\
            utils.idx(horizontalFences,y,x-1,False) |\
            utils.idx(horizontalFences,y,x,False) |\
            utils.idx(verticalFences,x,y-1,False) |\
            utils.idx(verticalFences,x,y,False)))
    # Shaded squares do not touch.
    for y in range(self.rows-1):
      for x in range(self.cols):
        require(~(shaded[y][x]&shaded[y+1][x]))
    for y in range(self.rows):
      for x in range(self.cols-1):
        require(~(shaded[y][x]&shaded[y][x+1]))
    # Require clue sums
    for y in range(self.rows):
      for x in range(self.cols):
        cell = self.board.getCell(x,y)
        if cell == None:
          continue
        direction = cell[0]
        count = cell[1]
        if direction == U:
          require(sum_bools(count, [shaded[i][x] for i in range(y)]))
        elif direction == D:
          require(sum_bools(count, [shaded[i][x] for i in range(y+1,self.rows)]))
        elif direction == L:
          require(sum_bools(count, [shaded[y][i] for i in range(x)]))
        elif direction == R:
          require(sum_bools(count, [shaded[y][i] for i in range(x+1,self.cols)]))

    num_solutions = solve(quiet=True)
    solution = {
      'horizontalFences': [[1 if str(x) == '1' else 0 for x in r] for r in horizontalFences],
      'verticalFences': [[1 if str(x) == '1' else 0 for x in r] for r in verticalFences],
    }
    return (num_solutions, solution)

  def decode(self):
    self.decodeArrowNumber16()


import math
import os
import random
import re
import sys

def matrixRotation(matrix, r):
    m = len(matrix)
    n = len(matrix[0])
    numLoops = int(min(m,n)/2)
    for loop in range(numLoops):
        perimeter = 2*(m+n-2)-8*loop
        rotations = r%perimeter
        for i in range(rotations):
            matrix = rotateLoopOnce(matrix, m, n, loop)
    print('\n'.join([' '.join(['{}'.format(item) for item in row])
      for row in matrix]))

def rotateLoopOnce(matrix, m, n, loop):
    botRow = m-1-loop
    rightCol = n-1-loop
    topLeft = matrix[loop][loop]
    topRight = matrix[loop][rightCol]
    botLeft = matrix[botRow][loop]
    botRight = matrix[botRow][rightCol]
    for i in range(n-(2*loop)-1):
        matrix[loop][loop+i] = matrix[loop][loop+i+1]
        matrix[botRow][rightCol-i] = matrix[botRow][rightCol-i-1]
    for j in range(m-(2*loop)-1):
        matrix[botRow-j][loop] = matrix[botRow-j-1][loop]
        matrix[loop+j][rightCol] = matrix[loop+j+1][rightCol]
    matrix[loop+1][loop] = topLeft
    matrix[loop][rightCol-1] = topRight
    matrix[botRow][loop+1] = botLeft
    matrix[botRow-1][rightCol] = botRight
    topLeft = 0
    topRight = 0
    botLeft = 0
    botRight = 0
    return matrix

if __name__ == '__main__':
    first_multiple_input = input().rstrip().split()

    m = int(first_multiple_input[0])

    n = int(first_multiple_input[1])

    r = int(first_multiple_input[2])

    matrix = []

    for _ in range(m):
        matrix.append(list(map(int, input().rstrip().split())))

    matrixRotation(matrix, r)


#!/usr/bin/env python
# coding: utf-8

# In[15]:

import numpy as np
from sklearn import tree,metrics,svm,ensemble,neighbors
import pandas as pd
from sklearn.model_selection import train_test_split,KFold
from sklearn import linear_model
import xgboost as xgb
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt

# ### 去掉缺失值较多的列

# In[20]:

nume = ['转炉终点温度', '转炉终点C', '转炉终点S',
        '钢水净重', '氮化钒铁FeV55N11-A', '低铝硅铁',
       '钒氮合金(进口)', '钒铁(FeV50-A)', '钒铁(FeV50-B)', '钒铁(FeV50-B).1', '硅铝钙',
       '硅铝合金FeAl30Si25', '硅铝锰合金球', '硅锰面（硅锰渣）', '硅铁(合格块)', '硅铁FeSi75-B',
       '石油焦增碳剂', '锰硅合金FeMn64Si27(合格块)', '锰硅合金FeMn68Si18(合格块)', '碳化硅(55%)',
       '硅钙碳脱氧剂']

# ### 缺失值处理、归一化

# In[31]:

CTrain = pd.read_excel('q1_1_收得率.xls')[nume+['C收得率']]
# CTrain = CTrain.dropna()
CTrain = CTrain.drop(CTrain[CTrain['C收得率'].isnull()].index)
# 离群值处理
outs=['转炉终点温度', '转炉终点C','钢水净重']
for out in outs:
    CTrain.drop(CTrain[(CTrain[out]>(CTrain[out].mean()+3*CTrain[out].std()))|(CTrain[out]<(CTrain[out].mean()-3*CTrain[out].std()))].index,inplace=True)

CTrain = CTrain.fillna(CTrain.mean())
X = CTrain[nume] #[['钢水净重','连铸正样C', '连铸正样Ceq_val', '低铝硅铁','石油焦增碳剂']]
X = X[nume].apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)+1e-10)).values
Y = CTrain[['C收得率']].values
len(CTrain.values)

# In[32]:

# cget = (CTrain['C收得率']-CTrain['C收得率'].min())/(CTrain['C收得率'].max()-CTrain['C收得率'].min())
plt.plot(X['钢水净重'][::50])
plt.plot(CTrain['C收得率'][::50])

# In[33]:

# #缺失值数量统计
# CTrain.isnull().sum()

# In[34]:

CTrain.head(1)

# In[35]:

# train_data, test_data, train_target, test_target = train_test_split(X.values,Y.values,test_size=0.2)

# ### 决策树回归

# In[36]:

clf = tree.DecisionTreeRegressor()
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index])
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# 以上三个值分别为R2决定系数、平均绝对误差、平均平方误差

# ### 线性回归

# In[49]:

clf = linear_model.LinearRegression()
ki = 4
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index])
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# In[50]:

mapk={}
for i in range(len(nume)):
    mapk[nume[i]]=np.round(clf.coef_[0][i],4)
    print(nume[i],' & ',np.round(clf.coef_[0][i],4))
# sorted(mapk.items(),key=lambda item:abs(item[1]))

# ### SVM

# In[115]:

clf = svm.SVR(gamma='auto')
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 贝叶斯

# In[116]:

clf = linear_model.BayesianRidge()
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 集成

# In[117]:

clf = ensemble.GradientBoostingRegressor(n_estimators=100, learning_rate=0.1,max_depth=1, random_state=0, loss='ls')
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 多项式回归（emm）

# In[142]:

clf = Pipeline([('poly', PolynomialFeatures(degree=2)),('linear', linear_model.LinearRegression(fit_intercept=False))])
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# # Mn元素预测

# ### 数据预处理

# In[123]:

nume = ['转炉终点温度', '转炉终点C', '转炉终点Mn', '转炉终点S',
       '转炉终点Si', '钢水净重', '连铸正样Mn',
#         '连铸正样C','连铸正样S', '连铸正样P', '连铸正样Si',
#        '连铸正样Ceq_val', '连铸正样Cr', '连铸正样Ni_val', '连铸正样Cu_val',
#        '连铸正样V_val', '连铸正样Alt_val', '连铸正样Als_val', '连铸正样Mo_val', '连铸正样Ti_val',
        '氮化钒铁FeV55N11-A', '低铝硅铁',
       '钒氮合金(进口)', '钒铁(FeV50-A)', '钒铁(FeV50-B)', '钒铁(FeV50-B).1', '硅铝钙',
       '硅铝合金FeAl30Si25', '硅铝锰合金球', '硅锰面（硅锰渣）', '硅铁(合格块)', '硅铁FeSi75-B',
       '石油焦增碳剂', '锰硅合金FeMn64Si27(合格块)', '锰硅合金FeMn68Si18(合格块)', '碳化硅(55%)',
       '硅钙碳脱氧剂']

# In[131]:

MnTrain = pd.read_excel('data1.xlsx')[nume]
# Mn初始含量较低，对反应收得率影响小，使用均值替换缺失值
MnTrain[['转炉终点Mn']] = MnTrain[['转炉终点Mn']].fillna(MnTrain[['转炉终点Mn']].mean())
# 去掉为对合金化后合金钢采样的数据
MnTrain = MnTrain.drop(MnTrain[MnTrain['连铸正样Mn'].isnull()].index)
# 使用均值替代少量的缺失值
MnTrain[nume] = MnTrain[nume].fillna(MnTrain[nume].mean())
# 重新计算Mn收得率
mn_t=['硅铝锰合金球','硅锰面（硅锰渣）','锰硅合金FeMn64Si27(合格块)','锰硅合金FeMn68Si18(合格块)']
mn_p = [0.3,0.664,0.664,0.664]
mn_total=(MnTrain[mn_t]*mn_p).sum(axis=1)
mn_comsu=(MnTrain['连铸正样Mn']-MnTrain['转炉终点Mn'])*MnTrain['钢水净重']
MnTrain['Mn收得率']=mn_comsu/(mn_total)
# 删掉异常数据
MnTrain = MnTrain.drop(MnTrain[MnTrain['Mn收得率']>1.5].index)

# 构造训练数据并对自变量值进行归一化
X = MnTrain[nume].apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)+1e-9))
X.drop('连铸正样Mn',axis=1,inplace=True)
X=X.values
Y = MnTrain[['Mn收得率']].values
# train_data, test_data, train_target, test_target = train_test_split(X.values,Y.values,test_size=0.2)
len(Y)

# In[132]:

# MnTrain #.isnull().sum()
# Y.sort_values('Mn收得率')

# ### 多项式拟合

# In[133]:

clf = Pipeline([('poly', PolynomialFeatures(degree=2)),('linear', linear_model.LinearRegression(fit_intercept=False))])
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# In[134]:

clf = linear_model.BayesianRidge()
clf = clf.fit(train_data,train_target.flatten())
pre_y = clf.predict(test_data)
metrics.r2_score(pre_y,test_target),metrics.mean_absolute_error(pre_y,test_target),metrics.mean_squared_error(pre_y,test_target)

# In[135]:

clf = linear_model.LinearRegression()
clf = clf.fit(train_data,train_target)
pre_y = clf.predict(test_data)
metrics.r2_score(pre_y,test_target),metrics.mean_absolute_error(pre_y,test_target),metrics.mean_squared_error(pre_y,test_target)

# In[ ]:


from zope.component import provideUtility
from agx.core.interfaces import IScope
from agx.core.metaconfigure import _chkregistered
from scope import (
    XMIScope,
    StereotypeScope,
)

def registerXMIScope(name, transform, tags, type, class_=XMIScope):
    name = '%s.%s' % (transform, name)
    _chkregistered(IScope, name=name)
    scope = class_(name, tags, type)
    provideUtility(scope, provides=IScope, name=name)

def registerStereotypeScope(name, transform, class_=StereotypeScope):
    name = '%s.%s' % (transform, name)
    _chkregistered(IScope, name=name)
    scope = class_()
    provideUtility(scope, provides=IScope, name=name)


from PyQt5.QtWidgets import (QWidget,QMainWindow, QApplication, QPushButton,
QRadioButton, QLabel, QGridLayout, QCheckBox, QLineEdit, QFileDialog, QFontDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QFileDialog)
import sys
from PyQt5.QtGui import QFont
from PyQt5.QtCore import QSize
from XImageClass import QHRadioGroupBox, QRadioButtonCustom

class App(QWidget):

    label_font = QFont('Consolas',10)

    def __init__(self):
        super().__init__()
        self.setWindowTitle('new-XImage')
        # self.setGeometry(500,200,700,500)
        # self.setWindowIcon()
        self.setupUI()

    def setupUI(self):

        self.grid_layout = QGridLayout(self)
        self.setLayout(self.grid_layout)

        self.label_find = QLabel('Find:', self)
        self.grid_layout.addWidget(self.label_find,0,0)

        self.line_find = QLineEdit(self)
        self.line_find.setPlaceholderText('What do you want to find?')
        self.grid_layout.addWidget(self.line_find,0,1)

        self.label_page = QLabel('Page:', self)
        self.grid_layout.addWidget(self.label_page,1,0)

        self.line_page = QLineEdit(self)
        self.grid_layout.addWidget(self.line_page,1,1)

        self.label_per_page = QLabel('Per page:', self)
        self.grid_layout.addWidget(self.label_per_page,2,0)

        self.line_per_page = QLineEdit(self)
        self.grid_layout.addWidget(self.line_per_page,2,1)

        self.label_save_at = QLabel('Save at:', self)
        self.grid_layout.addWidget(self.label_save_at,3,0)

        self.btt_save_at_browse = QPushButton('Browse...')
        self.btt_save_at_browse.clicked.connect(self.browse)
        self.grid_layout.addWidget(self.btt_save_at_browse,3,2)

        self.line_save_at = QLineEdit(self)
        self.grid_layout.addWidget(self.line_save_at,3,1)

        self.radio_group_box_quality = QHRadioGroupBox('Quality')
        self.rd_quality_raw = QRadioButtonCustom('Raw', 1, self)
        self.rd_quality_full = QRadioButtonCustom('Full', 2, self)
        self.rd_quality_regular = QRadioButtonCustom('Regular', 3, self)
        self.rd_quality_small = QRadioButtonCustom('Small', 4, self)
        self.rd_quality_thumnail = QRadioButtonCustom('Thumnail', 5, self)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_raw)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_full)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_regular)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_small)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_thumnail)
        self.grid_layout.addWidget(self.radio_group_box_quality,4,0,1,3)

        self.rd_group_box_order_by = QHRadioGroupBox('Order by',parent=self)
        self.rd_latest = QRadioButtonCustom('Latest', 1, self.rd_group_box_order_by)
        self.rd_oldest = QRadioButtonCustom('Oldest', 2, self.rd_group_box_order_by)
        self.rd_popular = QRadioButtonCustom('Popular', 3, self.rd_group_box_order_by)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_latest)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_oldest)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_popular)
        self.grid_layout.addWidget(self.rd_group_box_order_by,5,0,1,3)

        self.checkbox_random = QCheckBox('Random')
        self.checkbox_random.setToolTip('Get random images, maximum is 20 images')
        self.checkbox_random.stateChanged.connect(self.randomIsTrue)
        self.grid_layout.addWidget(self.checkbox_random,6,0)

        self.btt_start = QPushButton('Start')
        self.btt_start.clicked.connect(self.download)
        self.grid_layout.addWidget(self.btt_start,7,0,1,3)

        self.show()

    def download(self):
        # get info download
        pass

    def randomIsTrue(self,state):
        if state > 0:
            self.rd_group_box_order_by.setEnabled(False)
            self.line_per_page.setEnabled(False)
            self.line_page.setEnabled(False)
        else:
            self.rd_group_box_order_by.setEnabled(True)
            self.line_per_page.setEnabled(True)
            self.line_page.setEnabled(True)

    def browse(self):
        file = QFileDialog.getExistingDirectory(self,"Select a folder")
        self.line_save_at.setText(file)

if __name__ == "__main__":
    a = QApplication(sys.argv)
    app = App()
    sys.exit(a.exec_())


#Faça um Programa que leia 20 números inteiros e armazene-os num vetor.
#Armazene os números pares no vetor PAR e os números IMPARES no vetor impar.
#Imprima os três vetores.

numeros = []
total_numeros = 20
num_inicial = 1

while num_inicial <= total_numeros:
    numeros.append(int(input("Digite o número {} de {}: ".format(num_inicial, total_numeros))))
    num_inicial = num_inicial + 1

pares = []
impares = []
i = 0
posic_inicial = 0

while i < total_numeros:
    if numeros[posic_inicial] % 2 == 0:
        pares.append(numeros[posic_inicial])
    else:
        impares.append(numeros[posic_inicial])
    posic_inicial = posic_inicial + 1
    i = i + 1

print("Os números digitados foram:", numeros)
print("Os número pares são:", pares)
print("Os números ímpares são:", impares)


import torch
import torchvision
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets, transforms, models
import torchvision.models as models
from PIL import Image
import json
from matplotlib.ticker import FormatStrFormatter
import os
import cv2
import argparse
from pprint import pprint
parser = argparse.ArgumentParser()
parser.add_argument("t", type=str, help= 'specify a car type; type none if you do not have one')
args = parser.parse_args()
type = args.t
print(type + ' Car will be now searched for')
model = models.resnet34(pretrained=True)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 196)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
lrscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, threshold = 0.9)
def load_checkpoint(filepath):
    checkpoint = torch.load(filepath, map_location=torch.device('cpu'))

    # model.load_state_dict(checkpoint['state_dict'])
    model.load_state_dict(checkpoint['state_dict'], strict=False)
    model.class_to_idx = checkpoint['class_to_idx']

    return model

model = load_checkpoint('my_checkpoint1.pth')
model = torch.nn.DataParallel(model)

def process_image(image):
    # Process a PIL image for use in a PyTorch model

    # Converting image to PIL image using image file path
    pil_im = Image.open(image, mode='r').convert('RGB')

    # Building image transform
    transform = transforms.Compose([transforms.Resize((244, 244)),
                                    # transforms.CenterCrop(224),
                                    transforms.ToTensor(),
                                    transforms.Normalize([0.485, 0.456, 0.406],
                                                         [0.229, 0.224, 0.225])])

    # Transforming image for use with network
    pil_tfd = transform(pil_im)

    # Converting to Numpy array
    array_im_tfd = np.array(pil_tfd)

    return array_im_tfd

def predict(image_path, model, topk=5):
    # Implement the code to predict the class from an image file

    # Loading model - using .cpu() for working with CPUs
    loaded_model = load_checkpoint(model).cpu()
    # Pre-processing image
    img = process_image(image_path)
    # Converting to torch tensor from Numpy array
    img_tensor = torch.from_numpy(img).type(torch.FloatTensor)
    # Adding dimension to image to comply with (B x C x W x H) input of model
    img_add_dim = img_tensor.unsqueeze_(0)

    # Setting model to evaluation mode and turning off gradients
    loaded_model.eval()
    with torch.no_grad():
        # Running image through network
        output = loaded_model.forward(img_add_dim)

    # conf, predicted = torch.max(output.data, 1)
    probs_top = output.topk(topk)[0]
    predicted_top = output.topk(topk)[1]

    # Converting probabilities and outputs to lists
    conf = np.array(probs_top)[0]
    predicted = np.array(predicted_top)[0]

    # return probs_top_list, index_top_list
    return conf, predicted
def most_frequent(List):
    return max(set(List), key = List.count)
def find_classes(dir):
    classes = os.listdir(dir)
    classes.sort()
    class_to_idx = {classes[i]: i for i in range(len(classes))}
    return classes, class_to_idx
classes, c_to_idx = find_classes("car_data/train")
model_path = 'my_checkpoint1.pth'
classes_detect = []
count = 0
frame_Car = {}
for d in [d for d in os.listdir("Video/")]:
    count= count + 1
    img_path = "Video/" + d

    if (img_path != 'Video/.DS_Store'):
        if (count < 500):
            conf1, predicted1 = predict(img_path, model_path, topk=5)
            classes_detect.append(predicted1[0])
            frame_Car[count]= {}
            frame_Car[count]['file'] = img_path
            frame_Car[count]['ID'] = predicted1[0]

    print(img_path)
    print(count)
    print(classes[(most_frequent(classes_detect))])
pprint(frame_Car)
#Sedan
#SUV
#Coupe
#Convertible
#Cab
#Van
#Wagon
#Hatchback
#Minivan
img_file_arr = []
img_arr=[]
predicted_car = classes[(most_frequent(classes_detect))]
predicted_class = most_frequent(classes_detect)
for i in frame_Car.values():
    if(i['ID'] == predicted_class):
        img_file_arr.append([i['file']])
if (type != 'none'):
    if (type in predicted_car):
        print('Found type: ' + type + '   ')
        print(predicted_car)
    else:
        print('Did not find a ' + type + ', but found ' + ' ' + predicted_car)
else:
    print('found ' + predicted_car)
str1 = ""

for i in range(len(img_file_arr)):
    file = str1.join(img_file_arr[i])
    img = cv2.imread(file)
    height, width, layers = img.shape
    size = (width,height)
    img_arr.append(img)
fourcc = cv2.VideoWriter_fourcc(*'MP4V')

out = cv2.VideoWriter('project.mp4',fourcc, 3, size)
for i in range(len(img_arr)):
    out.write(img_arr[i])
out.release()


import abc
from collections import OrderedDict
import time

import gtimer as gt
import numpy as np
import ray

from rlkit.core import logger, eval_util
from rlkit.data_management.env_replay_buffer import MultiTaskReplayBuffer
from rlkit.data_management.path_builder import PathBuilder
from rlkit.samplers.in_place import InPlacePathSampler
from rlkit.torch import pytorch_util as ptu

class MetaRLAlgorithm(metaclass=abc.ABCMeta):
    def __init__(
            self,
            env,
            agent,
            train_goals,
            wd_goals,
            ood_goals,
            replay_buffers,
            meta_batch_size=64,
            num_iterations=100,
            num_train_steps_per_itr=1000,
            num_tasks=100,
            num_steps_prior=100,
            num_steps_posterior=100,
            num_extra_rl_steps_posterior=100,
            num_evals=10,
            num_steps_per_eval=1000,
            max_path_length=1000,
            discount=0.99,
            reward_scale=1,
            num_exp_traj_eval=1,
            eval_deterministic=True,
            render=False,
            save_replay_buffer=False,
            save_algorithm=False,
            save_environment=False,
            render_eval_paths=False,
            dump_eval_paths=False,
            plotter=None,
            use_same_context=True,
            recurrent=False,
    ):
        """
        :param env: training env
        :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in
        :param train_tasks: list of tasks used for training
        :param eval_tasks: list of tasks used for eval

        see default experiment config file for descriptions of the rest of the arguments
        """
        self.env = env
        self.agent = agent
        self.train_goals = train_goals
        self.wd_goals = wd_goals
        self.ood_goals = ood_goals
        self.replay_buffers = replay_buffers
        self.num_iterations = num_iterations
        self.num_train_steps_per_itr = num_train_steps_per_itr
        self.meta_batch_size = meta_batch_size
        self.num_evals = num_evals
        self.num_steps_per_eval = num_steps_per_eval
        self.max_path_length = max_path_length
        self.discount = discount
        self.reward_scale = reward_scale
        self.num_exp_traj_eval = num_exp_traj_eval
        self.eval_deterministic = eval_deterministic
        self.render = render
        self.save_replay_buffer = save_replay_buffer
        self.save_algorithm = save_algorithm
        self.save_environment = save_environment
        self.use_same_context = use_same_context
        self.recurrent=recurrent

        self.eval_statistics = None
        self.render_eval_paths = render_eval_paths
        self.dump_eval_paths = dump_eval_paths
        self.plotter = plotter

        self.sampler = InPlacePathSampler(
            env=env,
            policy=agent,
            max_path_length=self.max_path_length,
        )

        # separate replay buffers for
        # - training RL update
        # - training encoder update

        self._n_env_steps_total = 0
        self._n_train_steps_total = 0
        self._n_rollouts_total = 0
        self._do_train_time = 0
        self._epoch_start_time = None
        self._algo_start_time = None
        self._old_table_keys = None
        self._current_path_builder = PathBuilder()
        self._exploration_paths = []

    def make_exploration_policy(self, policy):
         return policy

    def make_eval_policy(self, policy):
        return policy

    def train(self):
        '''
        meta-training loop
        '''
        self.pretrain()
        gt.reset()
        gt.set_def_unique(False)
        self._current_path_builder = PathBuilder()

        # at each iteration, we first collect data from tasks, perform meta-updates, then try to evaluate
        for it_ in gt.timed_for(
                range(self.num_iterations),
                save_itrs=True,
        ):
            self._start_epoch(it_)
            self.training_mode(True)

            # Sample train tasks and compute gradient updates on parameters.
            batch_idxes = np.random.randint(
                    0, len(self.train_goals), size=self.meta_batch_size
            )
            train_batch_obj_id = self.replay_buffers.sample_training_data(
                batch_idxes, self.use_same_context
            )
            for _ in range(self.num_train_steps_per_itr):
                train_raw_batch = ray.get(train_batch_obj_id)
                gt.stamp('sample_training_data', unique=False)

                batch_idxes = np.random.randint(
                    0, len(self.train_goals), size=self.meta_batch_size
                )
                # In this way, we can start the data sampling job for the
                # next training while doing training for the current loop.
                train_batch_obj_id = self.replay_buffers.sample_training_data(
                    batch_idxes, self.use_same_context
                )
                gt.stamp('set_up_sampling', unique=False)

                train_data = self.construct_training_batch(train_raw_batch)
                gt.stamp('construct_training_batch', unique=False)

                self._do_training(train_data)
                self._n_train_steps_total += 1
            gt.stamp('train')

            self.training_mode(False)

            # eval
            self._try_to_eval(it_)
            gt.stamp('eval')

            self._end_epoch()
            if it_ == self.num_iterations:
                logger.save_itr_params(it_, self.agent.get_snapshot())

    def construct_training_batch(self, raw_batch):
        ''' Construct training batch from raw batch'''
        state = np.concatenate([rb[0] for rb in raw_batch], axis=0)
        next_state = np.concatenate([rb[1] for rb in raw_batch], axis=0)
        actions = np.concatenate([rb[2] for rb in raw_batch], axis=0)
        rewards = np.concatenate([rb[3] for rb in raw_batch], axis=0)
        dones = np.concatenate([rb[4] for rb in raw_batch], axis=0)
        contexts = np.concatenate([rb[5] for rb in raw_batch], axis=0)

        return [state, next_state, actions, rewards, dones, contexts]

    def pretrain(self):
        """
        Do anything before the main training phase.
        """
        pass

    def _try_to_eval(self, epoch):
        logger.save_extra_data(self.get_extra_data_to_save(epoch))
        if self._can_evaluate():
            self.evaluate(epoch)

            params = self.get_epoch_snapshot(epoch)
            logger.save_itr_params(epoch, params)

            table_keys = logger.get_table_key_set()
            if self._old_table_keys is not None:
                assert table_keys == self._old_table_keys, (
                    "Table keys cannot change from iteration to iteration."
                )
            self._old_table_keys = table_keys

            logger.record_tabular(
                "Number of train steps total",
                self._n_train_steps_total,
            )
            logger.record_tabular(
                "Number of env steps total",
                self._n_env_steps_total,
            )
            logger.record_tabular(
                "Number of rollouts total",
                self._n_rollouts_total,
            )

            times_itrs = gt.get_times().stamps.itrs
            train_time = times_itrs['train'][-1]
            sample_time = times_itrs['sample_training_data'][-1]
            eval_time = times_itrs['eval'][-1] if epoch > 0 else 0
            epoch_time = train_time + sample_time + eval_time
            total_time = gt.get_times().total

            logger.record_tabular('Train Time (s)', train_time)
            logger.record_tabular('(Previous) Eval Time (s)', eval_time)
            logger.record_tabular('Sample Time (s)', sample_time)
            logger.record_tabular('Epoch Time (s)', epoch_time)
            logger.record_tabular('Total Train Time (s)', total_time)

            logger.record_tabular("Epoch", epoch)
            logger.dump_tabular(with_prefix=False, with_timestamp=False)
        else:
            logger.log("Skipping eval for now.")

    def _can_evaluate(self):
        """
        One annoying thing about the logger table is that the keys at each
        iteration need to be the exact same. So unless you can compute
        everything, skip evaluation.

        A common example for why you might want to skip evaluation is that at
        the beginning of training, you may not have enough data for a
        validation and training set.

        :return:
        """
        # eval collects its own context, so can eval any time
        return True

    def _can_train(self):
        return True

    def _get_action_and_info(self, agent, observation):
        """
        Get an action to take in the environment.
        :param observation:
        :return:
        """
        agent.set_num_steps_total(self._n_env_steps_total)
        return agent.get_action(observation,)

    def _start_epoch(self, epoch):
        self._epoch_start_time = time.time()
        self._exploration_paths = []
        self._do_train_time = 0
        logger.push_prefix('Iteration #%d | ' % epoch)

    def _end_epoch(self):
        logger.log("Epoch Duration: {0}".format(
            time.time() - self._epoch_start_time
        ))
        logger.log("Started Training: {0}".format(self._can_train()))
        logger.pop_prefix()

    ##### Snapshotting utils #####

    def get_epoch_snapshot(self, epoch):
        data_to_save = dict(
            epoch=epoch,
            exploration_policy=self.agent,
        )
        if self.save_environment:
            data_to_save['env'] = self.env
        return data_to_save

    def get_extra_data_to_save(self, epoch):
        """
        Save things that shouldn't be saved every snapshot but rather
        overwritten every time.
        :param epoch:
        :return:
        """
        if self.render:
            self.env.render(close=True)
        data_to_save = dict(
            epoch=epoch,
        )
        if self.save_algorithm:
            data_to_save['algorithm'] = self
        if epoch == self.num_iterations - 1:
            data_to_save['algorithm'] = self
        return data_to_save

    def collect_paths(self, goal, epoch, run):
        self.env.set_goal(goal)

        self.agent.clear_z()
        paths = []
        num_transitions = 0
        num_trajs = 0
        while num_transitions < self.num_steps_per_eval:
            path, num = self.sampler.obtain_samples(deterministic=self.eval_deterministic, max_samples=self.num_steps_per_eval - num_transitions, max_trajs=1, accum_context=True)
            paths += path
            num_transitions += num
            num_trajs += 1
            if num_trajs >= self.num_exp_traj_eval:
                self.agent.infer_posterior(self.agent.context)

        if self.sparse_rewards:
            for p in paths:
                sparse_rewards = np.stack(e['sparse_reward'] for e in p['env_infos']).reshape(-1, 1)
                p['rewards'] = sparse_rewards

        goal = self.env._goal
        for path in paths:
            path['goal'] = goal # goal

        # save the paths for visualization, only useful for point mass
        if self.dump_eval_paths:
            logger.save_extra_data(paths, path='eval_trajectories/evla_goal{}-epoch{}-run{}'.format(goal, epoch, run))

        return paths

    def _do_eval(self, goal_set, epoch):

        final_returns = []
        final_achieved = []
        for goal in goal_set:
            all_rets = []
            all_achieved = []
            for r in range(self.num_evals):
                paths = self.collect_paths(goal, epoch, r)
                all_rets.append([eval_util.get_average_returns([p]) for p in paths])
                all_achieved.append([eval_util.get_average_achieved([p]) for p in paths])
            final_returns.append(np.mean([a[-1] for a in all_rets]))
            final_achieved.append(np.mean([a[-1] for a in all_achieved]))

        return final_returns, final_achieved

    def evaluate(self, epoch):
        if self.eval_statistics is None:
            self.eval_statistics = OrderedDict()

        ### sample trajectories from prior for debugging / visualization
        if self.dump_eval_paths:
            # 100 arbitrarily chosen for visualizations of point_robot trajectories
            # just want stochasticity of z, not the policy
            self.agent.clear_z()
            prior_paths, _ = self.sampler.obtain_samples(deterministic=self.eval_deterministic, max_samples=self.max_path_length * 20,
                                                        accum_context=False,
                                                        resample=1)
            logger.save_extra_data(prior_paths, path='eval_trajectories/prior-epoch{}'.format(epoch))

        ### train tasks
        # eval on a subset of train tasks for speed
        eval_util.dprint('evaluating on {} train tasks'.format(len(self.train_goals)))

        ### eval train tasks with on-policy data to match eval of test tasks
        train_final_returns, train_final_achieved = self._do_eval(self.train_goals, epoch)

        # Comment this line for walker-param
        # train_final_achieved_pair = [(train_final_achieved[i], goal) for i, goal in enumerate(self.train_goals)]
        train_final_achieved_pair = [(train_final_achieved[i], -1) for i, goal in enumerate(self.train_goals)]

        eval_util.dprint('train final achieved')
        eval_util.dprint(train_final_achieved_pair)

        ### WD tasks

        eval_util.dprint('evaluating on {} wd tasks'.format(len(self.wd_goals)))
        wd_final_returns, wd_final_achieved = self._do_eval(self.wd_goals, epoch)

        # Comment this line for walker-param
        # wd_final_achieved_pair = [(wd_final_achieved[i], goal) for i, goal in enumerate(self.wd_goals)]
        wd_final_achieved_pair = [(wd_final_achieved[i], -1) for i, goal in enumerate(self.wd_goals)]

        eval_util.dprint('WD test final achieved')
        eval_util.dprint(wd_final_achieved_pair)

        # ### OOD tasks

        # eval_util.dprint('evaluating on {} wd tasks'.format(len(self.ood_goals)))
        # ood_final_returns, ood_final_achieved = self._do_eval(self.ood_goals, epoch)

        # # Comment this line for walker-param
        # # ood_final_achieved_pair = [(ood_final_achieved[i], goal) for i, goal in enumerate(self.ood_goals)]
        # ood_final_achieved_pair = [(ood_final_achieved[i], -1) for i, goal in enumerate(self.ood_goals)]

        # eval_util.dprint('OOD test final achieved')
        # eval_util.dprint(ood_final_achieved_pair)

        # # save the final posterior
        # self.agent.log_diagnostics(self.eval_statistics)

        avg_train_return = np.mean(train_final_returns)
        avg_wd_return = np.mean(wd_final_returns)
        # avg_ood_return = np.mean(ood_final_returns)

        self.eval_statistics['AverageReturn_all_train_tasks'] = avg_train_return
        self.eval_statistics['AverageReturn_all_wd_tasks'] = avg_wd_return
        # self.eval_statistics['AverageReturn_all_ood_tasks'] = avg_ood_return

        self.eval_statistics['Return_all_train_tasks'] = train_final_returns
        self.eval_statistics['Return_all_wd_tasks'] = wd_final_returns
        # self.eval_statistics['Return_all_ood_tasks'] = ood_final_returns

        self.eval_statistics['Achieved_all_train_tasks'] = train_final_achieved_pair
        self.eval_statistics['Achieved_all_wd_tasks'] = wd_final_achieved_pair
        # self.eval_statistics['Achieved_all_ood_tasks'] = ood_final_achieved_pair

        for key, value in self.eval_statistics.items():
            logger.record_tabular(key, value)
        self.eval_statistics = None

        if self.plotter:
            self.plotter.draw()

    @abc.abstractmethod
    def training_mode(self, mode):
        """
        Set training mode to `mode`.
        :param mode: If True, training will happen (e.g. set the dropout
        probabilities to not all ones).
        """
        pass

    @abc.abstractmethod
    def _do_training(self, train_data):
        """
        Perform some update, e.g. perform one gradient step.
        :return:
        """
        pass


DISPLAY_DEPTH = 3
LABEL_CROP_LENGTH = 50
LABEL_MAX_LENGTH = 300
ACTUALS_MAX_LENGTH = LABEL_MAX_LENGTH // 10
MAX_ELEMENTS = 500
MAX_SAMPLE_TIME_MS = 12_000

INPUT_N = 3
CMP_ALGORITHM_I = 0
SHOW_FULL_LABELS = False

USER_STATE_STORAGE_TYPE = "session"


'''
Filename: save_to_hdf5.py
Python Version: 3.6.5
Project: Neutrophil Identifier
Author: Yang Liu
Created date: Sep 5, 2018 4:13 PM
-----
Last Modified: Oct 3, 2018 3:57 PM
Modified By: Yang Liu
-----
License: MIT
http://www.opensource.org/licenses/MIT
'''

import h5py
import os
import glob
from PIL import Image
import numpy as np
from random import shuffle
import tables as tb

def save_to_hdf5(path):
    """
    """
    TRAINING_DATA_PECENTAGE = 0.8
    TESTING_DATA_PECENTAGE = 0.1

    src_path = os.path.abspath(path)
    des_path = os.path.join(src_path, "..")
    des_path = os.path.join(des_path, "76_79_80.hdf5")

    files_path = os.path.join(src_path, "*.png")
    files = glob.glob(files_path)
    labels = [1 if "pos" in file_name else 0 for file_name in files]

    c = list(zip(files, labels))
    shuffle(c)
    files, labels = zip(*c)

    m = len(files)
    part_train = int(m * TRAINING_DATA_PECENTAGE)
    part_test = part_train + int(m * TESTING_DATA_PECENTAGE)
    x_train = files[0: part_train]
    x_test = files[part_train: part_test]
    x_val = files[part_test:]
    y_train = labels[0: part_train]
    y_test = labels[part_train: part_test]
    y_val = labels[part_test:]

    train_shape = (len(x_train), 299, 299, 3)
    test_shape = (len(x_test), 299, 299, 3)
    val_shape = (len(x_val), 299, 299, 3)

    hdf5_file = h5py.File(des_path, mode='w')
    m_train = len(x_train)
    m_test = len(x_test)
    m_val = len(x_val)

    hdf5_file.create_dataset("train_img", train_shape, np.uint8)
    hdf5_file.create_dataset("test_img", test_shape, np.uint8)
    hdf5_file.create_dataset("val_img", val_shape, np.uint8)

    hdf5_file.create_dataset("train_labels", (m_train,), np.uint8)
    hdf5_file.create_dataset("test_labels", (m_test,), np.uint8)
    hdf5_file.create_dataset("val_labels", (m_val,), np.uint8)

    hdf5_file.create_dataset("train_mean", train_shape[1:], np.float32)

    hdf5_file["train_labels"][...] = y_train
    hdf5_file["test_labels"][...] = y_test
    hdf5_file["val_labels"][...] = y_val

    mean = np.zeros(train_shape[1:], np.float32)

    for i in range(m_train):
        if i % 500 == 0 and i > 1:
            print("Train data: {}/{}".format(i, m_train))

        x = x_train[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["train_img"][i, ...] = img
        mean += img / float(m_train)

    for i in range(m_test):
        if i % 100 == 0 and i > 1:
            print("Test data: {}/{}".format(i, m_test))

        x = x_test[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["test_img"][i, ...] = img

    for i in range(m_val):
        if i % 100 == 0 and i > 1:
            print("Validation data: {}/{}".format(i, m_val))

        x = x_val[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["val_img"][i, ...] = img

    hdf5_file["train_mean"][...] = mean
    hdf5_file.close()

def save_to_hdf5_with_tables(path):
    """
    """
    TRAINING_DATA_PECENTAGE = 0.8
    TESTING_DATA_PECENTAGE = 0.1

    src_path = os.path.abspath(path)
    des_path = os.path.join(src_path, "..")
    des_path = os.path.join(des_path, "76_79_80.hdf5")

    files_path = os.path.join(src_path, "*.png")
    files = glob.glob(files_path)
    labels = [1 if "pos" in file_name else 0 for file_name in files]

    c = list(zip(files, labels))
    shuffle(c)
    files, labels = zip(*c)

    m = len(files)
    part_train = int(m * TRAINING_DATA_PECENTAGE)
    part_test = part_train + int(m * TESTING_DATA_PECENTAGE)
    x_train = files[0: part_train]
    x_test = files[part_train: part_test]
    x_val = files[part_test:]
    y_train = labels[0: part_train]
    y_test = labels[part_train: part_test]
    y_val = labels[part_test:]

    data_shape = (0, 299, 299, 3)

    hdf5_file = tb.open_file(des_path, mode='w')
    m_train = len(x_train)
    m_test = len(x_test)
    m_val = len(x_val)

    train_storage = hdf5_file.create_earray(
        hdf5_file.root, "train_img", tb.UInt8Atom(), shape=data_shape
    )
    test_storage = hdf5_file.create_earray(
        hdf5_file.root, "test_img", tb.UInt8Atom(), shape=data_shape
    )
    val_storage = hdf5_file.create_earray(
        hdf5_file.root, "val_img", tb.UInt8Atom(), shape=data_shape
    )

    hdf5_file.create_array(hdf5_file.root, "train_labels", y_train)
    hdf5_file.create_array(hdf5_file.root, "test_labels", y_test)
    hdf5_file.create_array(hdf5_file.root, "val_labels", y_val)

    mean_storage = hdf5_file.create_earray(
        hdf5_file.root, "train_mean", tb.Float32Atom(), shape=data_shape
    )

    mean = np.zeros(data_shape[1:], np.float32)

    for i in range(m_train):
        if i % 500 == 0 and i > 1:
            print("Train data: {}/{}".format(i, m_train))

        x = x_train[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        train_storage.append(img[None])
        mean += img / float(m_train)

    for i in range(m_test):
        if i % 100 == 0 and i > 1:
            print("Test data: {}/{}".format(i, m_test))

        x = x_test[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        test_storage.append(img[None])

    for i in range(m_val):
        if i % 100 == 0 and i > 1:
            print("Validation data: {}/{}".format(i, m_val))

        x = x_val[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        val_storage.append(img[None])

    mean_storage.append(mean[None])
    hdf5_file.close()

if __name__ == "__main__":
    import timeit
    start_time = timeit.default_timer()
    home = os.path.abspath("..")
    pool_path = os.path.join(home, "data", "pool")
    save_to_hdf5_with_tables(pool_path)
    end_time = timeit.default_timer()

    print("Time consumed: {}".format(end_time - start_time))


import torch
import torch.nn as nn
from typing import Callable, Union
from torchtyping import patch_typeguard
from einops import rearrange
import timm
import open_clip
from functools import partial

# ----------------------------- Utils --------------------------------------

# clip.model.LayerNorm = (
#     nn.LayerNorm
# )  # we need to patch this for clip to work with deepspeed
# patch_typeguard()  # needed for torchtyping typechecks to work

class Lambda(torch.nn.Module):
    def __init__(self, fn: Callable):
        super().__init__()
        assert hasattr(fn, "__call__")
        self.fn = fn

    def forward(self, x):
        return self.fn(x)

# ------------------------- Image encoders ----------------------------------

def nfresnet50(
    device: Union[torch.device, str] = None, pretrained: bool = True
) -> nn.Module:
    """
    Loads nfresnet50 model, removing the pooling layer and replacing it with
    an adaptive pooling layer.
    """
    encoder = torch.nn.Sequential(
        *list(timm.create_model("nf_resnet50", pretrained=pretrained).children())[:-1]
    )
    pooling = torch.nn.AdaptiveAvgPool2d((1, 1))
    encoder = torch.nn.Sequential(encoder, pooling)
    if device is not None:
        encoder = encoder.to(device)
    return encoder

def clip_encoder(
    device: Union[torch.device, str] = None, name: str = "clip",
) -> nn.Module:
    """
    Loads clip's image encoder module, discarding the lm component.

    If the variant is a resnet model, we also remove the attention pooling.
    """
    if name in ["clip", "ViT-B/32"]:
        name, pretrained = "ViT-B-32", "openai"
    elif name in ["clip_resnet", "RN50x4"]:
        name, pretrained = "RN50x4", "openai"
    elif name in ["clip_resnet_large", "RN50x16"]:
        name, pretrained = "RN50x16", "openai"
    elif "openclip" in name:
        if "H" in name:
            name, pretrained = "ViT-H-14", "laion2b_s32b_b79k"
        elif "B" in name and "32" in name:
            name, pretrained = "ViT-B-32", "laion2b_s34b_b79k"
        else:
            raise NotImplementedError(f"Encoder {name} not recognized")
    else:
        raise NotImplementedError(f"Encoder {name} not recognized")

    # TODO better internet connection
    encoder = open_clip.create_model(name, device=device, precision="fp16" if "cuda" in str(device) else "fp32").visual  # , pretrained=pretrained).visual

    if "RN" in name:
        # remove attention pooling
        encoder.attnpool = Lambda(
            partial(rearrange, pattern="b d h w -> b (h w) d")
        )  # remove attn pooling, just use reshaped features

    if False and hasattr(encoder, "transformer"):  # TODO when do we want to disable pooling?
        def forward(self, x: torch.Tensor):
            x = self.conv1(x)  # shape = [*, width, grid, grid]
            x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
            x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
            x = torch.cat(
                [self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
                 x], dim=1)  # shape = [*, grid ** 2 + 1, width]
            x = x + self.positional_embedding.to(x.dtype)

            ## a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in
            # x = self.patch_dropout(x)
            x = self.ln_pre(x)

            x = x.permute(1, 0, 2)  # NLD -> LND
            x = self.transformer(x)
            x = self.ln_post(x)
            x = x.permute(1, 0, 2)  # LND -> NLD
            return x
        encoder.forward = partial(forward, encoder)

    if device is not None:
        encoder = encoder.to(device)

    return encoder

def get_image_encoder(
    name: str, device: Union[torch.device, str] = None, pretrained: bool = False
) -> torch.nn.Module:
    """
    Loads image encoder module
    """
    if name == "nfresnet50":
        encoder = nfresnet50(device=device, pretrained=pretrained)
    elif "clip" in name:
        encoder = clip_encoder(device=device, name=name)
    else:
        raise ValueError(f"image encoder {name} not recognized")
    return encoder


from sklearn.ensemble import RandomForestClassifier
import random
import import_data as impd
import pdb

#logic
def create_random_forest(num_trees,criterion,bootstrap,vectors,classes,max_depth):
    rfc=RandomForestClassifier(n_estimators=num_trees,bootstrap=bootstrap,criterion=criterion,max_depth=max_depth)
    rfc.fit(vectors,classes)
    return rfc

def predict_data(testing_data,clf):
    fvs=testing_data["vectors"]
    class_results=testing_data["classes"]
    predictions=clf.predict(fvs)
    total_vectors=len(fvs)
    idx=0
    correct_results=0
    results_by_class={
        "exp": {},
        "total": {}
    }
    for item in set(class_results):
        results_by_class["total"][item]=0
    for item in set(predictions):
        results_by_class["exp"][item]=0

    for _ in predictions:
        results_by_class["total"][class_results[idx]] += 1
        if(class_results[idx]==predictions[idx]):
            correct_results += 1
            results_by_class["exp"][predictions[idx]] += 1

        #print("Actual: " + class_results[idx] + ", result: " + predictions[idx])
        idx += 1
    return correct_results,total_vectors,results_by_class

def split_data(feature_vectors,classes):
    percent_split = 1
    training_vectors = list()
    training_classes=list()

    testing_vectors = list()
    testing_classes = list()
    idx=0
    for _ in feature_vectors:
        z=random.randint(1,10)
        if z <= percent_split:
            testing_vectors.append(feature_vectors[idx])
            testing_classes.append(classes[idx])
        else:
            training_vectors.append(feature_vectors[idx])
            training_classes.append(classes[idx])
        idx += 1
    return {"vectors":training_vectors,"classes":training_classes}, {"vectors":testing_vectors,"classes":testing_classes}


from .models import Navbar, Footer, BasePage
from django.conf import settings

def add_variable_to_context(request):
    navbar = Navbar.objects.first()
    footer = Footer.objects.first()
    base = BasePage.objects.first()
    data = {
        "user": request.user
    }
    return {
        'data': data,
        'nav': navbar,
        'footer': footer,
        'base' : base
    }


import option_keyboard.envs
from option_keyboard.option_keyboard.ok import option_keyboard
from option_keyboard.core.utils import set_global_seed
from option_keyboard.core.value_function import ValueFunction
import argparse
import gym
import torch
import numpy as np
import os
import pickle

parser = argparse.ArgumentParser('test_ok')
parser.add_argument('-e', '--env-name', default='ForagingWorld-v0',
                    help='Name of environment')
parser.add_argument('-s', '--seed', default=0, type=int,
                    help='Random seed')
parser.add_argument('--exp-name', required=True,
                    help='Name of experiment')
parser.add_argument('--n-test-episodes', default=100,
                    help='Number of test episodes')
parser.add_argument('--visualize', action='store_true',
                    help='Flag for visualization')
parser.add_argument('--saved-models', required=True,
                    help='Path to saved models')
parser.add_argument('--save-path', default='',
                    help='Path to file to which results are to be saved')
parser.add_argument('--w', default=[1, 1], nargs=2, type=int,
                    help='Weight vector')

def main():
    args = parser.parse_args()
    env = gym.make(args.env_name)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    set_global_seed(args.seed)

    d = env.num_resources()
    w = np.array(args.w)
    env.set_learning_options(w=w, flag=True)

    hyperparams_file = open(os.path.join(args.saved_models.split('saved_models')[0],
                            'hyperparams'), 'rb')

    # Loading saved models and constant values
    returns = []
    if args.save_path:
        fp = open(args.save_path, 'a+b')

    hyperparams = pickle.load(hyperparams_file)
    gamma = hyperparams.gamma_ok
    max_ep_steps = hyperparams.max_steps_agent

    value_fns = [ValueFunction(input_dim=env.observation_space.shape[0] + d,
                               action_dim=(env.action_space.n + 1),
                               n_options=d,
                               hidden=[64, 128],
                               batch_size=hyperparams.ok_batch_size,
                               gamma=gamma,
                               alpha=hyperparams.alpha_ok)
                 for _ in range(d)]

    for i in range(env.num_resources()):
        if not torch.cuda.is_available():
            checkpoint = torch.load(os.path.join(args.saved_models,
                                                 'value_fn_%d.pt' %
                                                 (i + 1)),
                                    map_location=torch.device('cpu'))
        else:
            checkpoint = torch.load(os.path.join(args.saved_models,
                                                 'value_fn_%d.pt' %
                                                 (i + 1)))

        value_fns[i].q_net.load_state_dict(checkpoint['Q'])
        value_fns[i].q_net.to(device)

    # Testing
    for _ in range(args.n_test_episodes):
        s = env.reset()
        done = False
        s = torch.from_numpy(s).float().to(device)
        n_steps = 0
        ret = 0

        while not done:
            (s_next, done, _, _, n_steps, info) = option_keyboard(env, s, w,
                                                                  value_fns,
                                                                  gamma,
                                                                  n_steps,
                                                                  max_ep_steps,
                                                                  device,
                                                                  args.visualize)

            ret += sum(info['rewards'])
            s = torch.from_numpy(s_next).float().to(device)

        print('Episode return:', ret)
        returns.append(ret)

    returns = np.array(returns)
    print('Mean: %f, Std. dev: %f' % (returns.mean(), returns.std()))
    pickle.dump({'Seed': args.seed, 'Returns': returns}, fp)
    fp.close()

if __name__ == '__main__':
    main()


## File contains all settings for
## Bloodytrinkets
##

# graph_colours needs colours for all ilevels!
graph_colours = {
  "880": "#4572a7",
  "890": "#aa4643",
  "900": "#89a54e",
  "910": "#71588f",
  "920": "#4198af",
  "930": "#db843d",
  "940": "#00E676"
}
# Defines itemlevels that shall be simed, ordered from highest to lowest (graph
# output will have this order reversed)
ilevels = [
  "940",
  "930",
  "920",
  "910",
  "900",
  "890",
  "880"
]

legendary = True
legendary_colour = "#ffeb3b"
legendary_ilevel = "970"

output_screen = False
# "json", "highchart"
output_types = ["json", "highchart"]

simc_settings = {}
simc_settings["simc"]         = "../simc.exe"
simc_settings["fight_styles"] = ["patchwerk", "helterskelter"]
simc_settings["iterations"]   = "250000"
simc_settings["target_error"] = "0.1"
simc_settings["threads"]      = ""
simc_settings["tier"]         = "T20M"

graph_title = "Shaman - Elemental - Patchwerk"
simc_settings["class"] = "shaman"
simc_settings["spec"]  = "elemental"

# Be aware that this feature basically makes the trinket disappear in the graph
# but has the benefit of showing a bit more realistic trinket values
simc_settings["use_second_trinket"] = False
simc_settings["second_trinket"] = ( "141482", "880" )     # ( id s, ilevel s),
                                                          # this is a int-vers stat stick

# You want to use a custom profile? Set c_profile to True and add a relative
# path and name
simc_settings["c_profile"]      = False
simc_settings["c_profile_path"] = "example_dir/"
simc_settings["c_profile_name"] = "example_name.simc"

simc_settings["ptr"] = False


import cv2
import os
import random

dataset_path = "/Users/jingwenshi/Desktop/Dataset/"

# Create Train, Test, and Valid folders with subfolders
folders = ['Train', 'Test', 'Valid']
subfolders = ['12%', '37%', '62%', '87%']

for folder in folders:
    for subfolder in subfolders:
        os.makedirs(os.path.join(folder, subfolder), exist_ok=True)

# Loop through all files in the directory
for video in os.listdir(dataset_path):
    # Check if the file has a .MOV extension
    if video.endswith(".MOV"):
        path = os.path.join(dataset_path, video)
        cam = cv2.VideoCapture(path)

        video_name = video[:-4]

        try:
            if not os.path.exists("temp_data"):
                os.makedirs("temp_data")
        except OSError:
            print('Error: Creating directory of data')

        currentframe = 0
        images = []

        while True:
            ret, frame = cam.read()

            if ret:
                if currentframe % 15 == 0:
                    name = f'./temp_data/{video_name}_' + str(currentframe) + '_' + video_name[-3:] + '.jpg'
                    print('Creating...' + name)
                    cv2.imwrite(name, frame)
                    images.append(name)
                currentframe += 1

            else:
                break

        cam.release()

        # Shuffle the images and assign them to Train, Test, and Valid folders
        random.shuffle(images)
        num_images = len(images)
        test_images = images[:num_images // 5]
        valid_images = images[num_images // 5:2 * num_images // 5]
        train_images = images[2 * num_images // 5:]

        for img_path in train_images + test_images + valid_images:
            img_name = os.path.basename(img_path)
            folder = "Train" if img_path in train_images else "Test" if img_path in test_images else "Valid"
            subfolder = img_name.split('_')[2][:3]  # Extract the percentage from the image name
            print(subfolder)
            dest_path = os.path.join(folder, subfolder, img_name)
            os.rename(img_path, dest_path)

# Remove the temp_data folder
os.rmdir("temp_data")


#!/usr/bin/env python3

from asyncio import all_tasks, current_task, gather, get_event_loop as loop, create_task
import websockets
from signal import SIGTERM, SIGINT

from .client import Client
from .game import Game

class Server:
    def __init__(self, port, updating_command=None, update_cmd=None, ping_every=120):
        self.port = port
        self.client_id = 1
        self.game = Game()
        if updating_command and not update_cmd:
            raise ValueError('update_cmd not specified')
        self.updating_command = updating_command
        self.update_cmd = update_cmd
        self.ping_every = ping_every

    async def caught(self, ws, path):
        cc = Client(self, self.game, ws, self.client_id)
        self.client_id += 1
        self.game.add(cc)
        await cc.handle()

    async def shutdown(self, sig=None):
        if sig:
            print(f'[SIGNAL] Got {sig}, closing connections...')
        await self.game.kill()
        tasks = [t for t in all_tasks() if t is not current_task()]
        [task.cancel() for task in tasks]
        print(f'[SIGNAL] Cancelling tasks...')
        results = await gather(*tasks, return_exceptions=True)
        for r in results:
            if isinstance(r, Exception):
                print(f'[SIGNAL] Got exception: {r}')
        print(f'[SIGNAL] Stopping...')
        loop().stop()

    def start(self):
        start_server = websockets.serve(self.caught, "", self.port)

        for sig in (SIGTERM, SIGINT):
            loop().add_signal_handler(sig, lambda sig=sig: create_task(self.shutdown(sig)))

        loop().run_until_complete(start_server)
        loop().run_forever()


from NormalSpider.DevTools.method_class import *

def open_dev_tools():
    base_url = "http://localhost:9000"

    mr = MakeRandom()
    time_random = mr.way_1_time()

    dev = Devtools()
    random_term = "/json/list?t="
    target_url = dev.make_url(base_url, random_term)
    url = dev.make_url(target_url, time_random)
    data = dev.get_page(url, "json")

    EZAccess_url = base_url + data[0]['devtoolsFrontendUrl']
    page = dev.get_page(EZAccess_url, "html")
    dev.save_it(EZAccess_url, page)
    print(page.text)

if __name__ == '__main__':
    open_dev_tools()


if __name__ == '__main__':
  society_links = []
  flat_urls = []
  with open('society.txt') as societyf:
    file_society_links = societyf.readlines()
    for line in file_society_links:
      society_links.append(line.replace('\n', ''))
  with open('flat.txt') as flatf:
    file_flat_urls = flatf.readlines()
    for line in file_flat_urls:
      flat_urls.append(line.replace('\n', ''))
  flat_urls.sort()
  society_links.sort()
  society_file = open('society_sorted.txt', 'w')
  for society_url in society_links:
    society_file.write(society_url)
    society_file.write('\n')
  society_file.close()
  flat_file = open('flat_sorted.txt', 'w')
  for flat_url in flat_urls:
    flat_file.write(flat_url)
    flat_file.write('\n')
  flat_file.close()


import os
import tempfile

from infra_buddy.context.deploy_ctx import DeployContext

from infra_buddy.aws import s3
from infra_buddy.aws.s3 import S3Buddy, CloudFormationDeployS3Buddy
from infra_buddy.deploy.s3_deploy import S3Deploy
from testcase_parent import ParentTestCase

class TestCloudFormationBuddy(object):

    def __init__(self, deploy_ctx):
        super(TestCloudFormationBuddy, self).__init__()
        self.ctx = deploy_ctx

    def get_export_value(self, param):
        return "{bucket_name}/{root_path}".format(
            root_path=self.ctx.cf_deploy_resource_path,
            bucket_name=self.ctx.cf_bucket_name
        )

class S3TestCase(ParentTestCase):
    def tearDown(self):
        pass

    @classmethod
    def setUpClass(cls):
        super(S3TestCase, cls).setUpClass()

    def test_bucket_configuration(self):
        self._validate_s3(self.test_deploy_ctx)

    def test_bucket_configuration_us_east_1(self):
        test_deploy_ctx = DeployContext.create_deploy_context(
            application="foo", role="bar-{}".format(self.run_random_word), environment="unit-test",
            defaults=self.east_config
        )
        self._validate_s3(test_deploy_ctx)

    def _validate_s3(self, deploy_ctx):
        s3_buddy = CloudFormationDeployS3Buddy(deploy_ctx)
        try:
            self.assertEqual(s3_buddy.key_root_path, self.test_deploy_ctx.cf_deploy_resource_path,
                             "Did not init correct path")
            self.assertEqual(s3_buddy.bucket.name, self.test_deploy_ctx.cf_bucket_name,
                             "Did not init correct bucket")
            if deploy_ctx.region == 'us-east-1':
                base = 's3'
            else:
                base = "s3-{}".format(deploy_ctx.region)
            self.assertEqual(s3_buddy.url_base,
                             "https://{}.amazonaws.com/unit-test-foo-cloudformation-deploy-resources".format(
                                 base),
                             "Did not init correct url")
        finally:
            self.clean_s3(s3_buddy)

    def test_file_operations(self):

        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        try:
            changeset = ParentTestCase._get_resource_path("cloudformation/sample_changeset.json")
            s3_buddy.upload(changeset)
            with open(changeset, 'r') as cs:
                change_set_string = cs.read()
                self.assertEqual(
                    s3_buddy.get_file_as_string('sample_changeset.json'),
                    change_set_string, "Failed to get file as string"
                )
            self.assertEqual(
                s3_buddy._get_s3_object('sample_changeset.json')['ContentType'],
                'application/json',
                "Did not persist correct content type"
            )
        finally:
            self.clean_s3(s3_buddy)

    def test_zip_download(self):
        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        compress = ParentTestCase._get_resource_path("s3_tests/test_compress.json.zip")
        s3_buddy.upload(compress)
        s3_url = "s3://{bucket}/{key}".format(
            bucket=self.test_deploy_ctx.cf_bucket_name,
            key=s3_buddy._get_upload_bucket_key_name(file=None, key_name="test_compress.json.zip")
        )
        temp_dir = tempfile.mkdtemp()
        test_file = os.path.join(temp_dir, "test_compress.json")
        try:
            s3.download_zip_from_s3_url(s3_url, temp_dir)
            self.assertTrue(os.path.exists(test_file), "Failed to decompress file")
        finally:
            self.clean_dir(temp_dir)
            self.clean_s3(s3_buddy)

    def test_s3_deploy(self):
        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        try:
            compress = ParentTestCase._get_resource_path("s3_tests/test_compress.json.zip")
            s3_buddy.upload(compress)
            s3_url = "{bucket}/{key}".format(
                bucket=self.test_deploy_ctx.cf_bucket_name,
                key=s3_buddy._get_upload_bucket_key_name(file=None, key_name="test_compress.json.zip")
            )
            # s3 deploy appends zip to it
            s3d = S3Deploy(
                artifact_id="test_compress.json", location=s3_url[:s3_url.rfind("/")], ctx=self.test_deploy_ctx
            )
            s3d.cloud_formation_buddy = TestCloudFormationBuddy(self.test_deploy_ctx)
            s3d.do_deploy()
            self.assertEqual(
                s3_buddy.get_file_as_string('test_compress.json'),
                "test-compress",
                "Failed to get file as string"
            )
        finally:
            self.clean_s3(s3_buddy)


import pandas as pd
import numpy as np
import pdb
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from mimic_paths import mimic_iv_path

ICD_CODE_FIELD = 'icd_code'
mimic_iv_data_path = mimic_iv_path
MODE_GROUP_MAP = {'ethnicity': ('BLACK/AFRICAN AMERICAN', 'WHITE', 'ASIAN', 'HISPANIC/LATINO'),
                  'insurance': ('Medicaid', 'Medicare'),
                  'marital_status': ('DIVORCED', 'MARRIED', 'SINGLE')}

prettify_group_name = {'BLACK/AFRICAN AMERICAN': 'Black/African American',
                        'WHITE': 'White',
                        'ASIAN': 'Asian',
                        'HISPANIC/LATINO': 'Hispanic/Latino',
                        'F': 'Female',
                        'M': 'Male',
                        'Medicaid': 'Medicaid',
                        'Medicare': 'Medicare',
                        'DIVORCED': 'Divorced',
                        'MARRIED': 'Married',
                        'SINGLE': 'Single'}

prettify_category_name = {'ethnicity': 'Ethnicity',
                          'insurance': 'Insurance',
                          'marital_status': 'Marital Status'}

# Helper functions for querying IDs
def get_ids_with_icd_codes(diagnoses, id_type, codes):
    ids = set(diagnoses.loc[diagnoses[ICD_CODE_FIELD].map(lambda x:any([x.startswith(code)
                                                                        for code in codes])), id_type])
    return ids

def get_ids_with_kws(diagnoses, id_type, kws, descr_field='long_title'):
    ids = set(diagnoses.loc[diagnoses[descr_field].map(lambda x:any([keyword in x.lower()
                                                                     for keyword in kws])), id_type])
    return ids

def get_idxs_of_group(ids, group_name, category, id_type='hadm_id'):
    # Function that returns idxs of a subset of ids that belong to group
    # specified by attr_name and attr_val
    # attr_name is some option in ['ethnicity']
    admissions = pd.read_csv(mimic_iv_data_path + 'core/admissions.csv.gz')
    group_ids = admissions[admissions[category] == group_name][id_type]
    group_ids = sorted(list(set(ids).intersection(set(group_ids))))

    id_to_index = {h_id : idx for idx, h_id in enumerate(ids)}
    group_id_idxs = [id_to_index[g_id] for g_id in group_ids]
    return group_id_idxs

def get_idxs_not_of_group(ids, group_name, category, id_type='hadm_id'):
    group_idxs = get_idxs_of_group(ids, group_name, category, id_type)
    all_idxs = list(range(len(ids)))
    not_group_idxs = sorted(list(set(all_idxs).difference(set(group_idxs))))
    return not_group_idxs

#     not_group_ids = sorted(list(set(ids).difference(set(group_ids))))
#     pdb.set_trace()
#     id_to_index = {h_id : idx for idx, h_id in enumerate(ids)}
#     not_group_id_idxs = [id_to_index[g_id] for g_id in not_group_ids]
#     return not_group_id_idxs

def get_ids_of_ethnicity(ids, ethnicity, id_type='hadm_id'):
    return get_ids_of_group(ids, ethnicity, 'ethnicity', id_type='hadm_id')

def preprocess_mimic_data(x, y, s, hadm_ids, subject_ids, groups, expmt_config, random_seed=0):
    if len(hadm_ids.shape) == 2:
        hadm_ids = np.squeeze(hadm_ids, 1)
    if len(hadm_ids.shape) == 2:
        subject_ids = np.squeeze(subject_ids, 1)
    if not len(y):
        y = s

    # Create group indicator columns
    n_groups = len(groups)
    group_indicators_list = []
    for group in groups:
        group_indicators = np.zeros((hadm_ids.shape[0], 1))
        group_idxs = get_idxs_of_group(hadm_ids, group, expmt_config['category'])
        group_indicators[group_idxs] = 1
        group_indicators_list.append(group_indicators)

    # Add group features to x
    x = np.concatenate([*group_indicators_list, x], axis=1)

    group_columns = list(range(n_groups))
    columns = group_columns + ['hadm_id', 'subject_id', 's']
    hadm_id_columns = np.stack([[x.flatten() for x in group_indicators_list] + [hadm_ids, subject_ids, s]])[0]
    hadm_id_df = pd.DataFrame(hadm_id_columns.T,  columns=columns)

    # Split patients into train, val, and test sets
    subject_id_df = hadm_id_df[['subject_id', 's'] + group_columns].copy()
    subject_id_df = subject_id_df.drop_duplicates(subset='subject_id', keep='first')
    subject_id_df['dummy'] = 1
    if expmt_config['stratify_mode'] == 'group_and_s':
        subject_id_df['stratify_var'] = subject_id_df[group_columns + ['s']].astype(str).agg('-'.join, axis=1)
    elif expmt_config['stratify_mode'] == 's':
        subject_id_df['stratify_var'] = subject_id_df['s']
    elif expmt_config['stratify_mode'] == 'group':
        subject_id_df['stratify_var'] = subject_id_df[group_columns + ['s']].astype(str).agg('-'.join, axis=1)
    elif expmt_config['stratify_mode'] == 'none':
        subject_id_df['stratify_var'] = subject_id_df['dummy']

    train_sids, test_sids = train_test_split(subject_id_df.subject_id,
                                                              test_size=.4,
                                                              shuffle=True, random_state=random_seed,
                                                              stratify=subject_id_df.stratify_var)
    test_sid_df = subject_id_df[subject_id_df['subject_id'].isin(test_sids)]
    val_sids, test_sids = train_test_split(test_sid_df.subject_id, test_size=.5,
                                           shuffle=True, random_state=random_seed,
                                           stratify=test_sid_df.stratify_var)

    # Map patient IDs to hospital admission IDs
    train_hadm_ids = hadm_id_df[hadm_id_df['subject_id'].isin(train_sids)]['hadm_id']
    val_hadm_ids = hadm_id_df[hadm_id_df['subject_id'].isin(val_sids)]['hadm_id']
    test_hadm_ids = hadm_id_df[hadm_id_df['subject_id'].isin(test_sids)]['hadm_id']

    # Map hospital admission IDs to indices in the original input, x
    hadm_id_to_idx = {h_id : idx for idx, h_id in enumerate(hadm_ids)}
    train_idxs = [hadm_id_to_idx[hadm_id] for hadm_id in list(train_hadm_ids)]
    val_idxs = [hadm_id_to_idx[hadm_id] for hadm_id in list(val_hadm_ids)]
    test_idxs = [hadm_id_to_idx[hadm_id] for hadm_id in list(test_hadm_ids)]

    # Define objects for train, val, and test set
    train = x[train_idxs], y[train_idxs], s[train_idxs]
    val = x[val_idxs], y[val_idxs], s[val_idxs]
    test = x[test_idxs],  y[test_idxs], s[test_idxs]
    return train, val, test

def normalize_x(x_train, x_val, x_test, normalize_groups=False, n_groups=2):
    scaler = StandardScaler()
    if normalize_groups:
        scaler.fit(x_train)
        x_train_norm, x_val_norm, x_test_norm = [scaler.transform(d) for d in (x_train, x_val, x_test)]
    else:
        scaler.fit(x_train[:,n_groups:])
        x_train_norm, x_val_norm, x_test_norm = [scaler.transform(d[:,n_groups:]) for d in (x_train, x_val, x_test)]
        norm_ds = []
        for norm_d, d in [(x_train_norm, x_train), (x_val_norm, x_val), (x_test_norm, x_test)]:
            norm_d_with_group = np.concatenate([d[:,:n_groups], norm_d], axis=1)
            norm_ds.append(norm_d_with_group)
        x_train_norm, x_val_norm, x_test_norm = norm_ds
    return (x_train_norm, x_val_norm, x_test_norm)

def get_group_weights(x, n_groups):
    group_weights = np.sum(x[:,:n_groups], axis=0)
    group_weights = np.max(group_weights) / group_weights
    group_weights = group_weights/np.sum(group_weights)
    group_weights = np.array(group_weights).flatten()
    return group_weights

# Helper functions for querying symptoms via ICD codes or keywords
def get_icd_code_long_title(names, code):
    code_names = names[names['icd_code'].str.startswith(code)]['long_title']
    return list(code_names)[0]

def get_icd_codes_with_prefix(names, code):
    code_names = names[names['icd_code'].str.startswith(code)]['icd_code']
    return list(code_names)

def get_coocurring_symptoms_codes(diagnoses, id_type, codes, key='long_title', print_output=True):
    assert type(codes) is list
    assert id_type in ['hadm_id', 'subject_id']
    # ids with ICD9 codes that contain any keyword in query
    ids = get_ids_with_icd_codes(diagnoses, id_type, codes)
    # all diagnoses associated with ids
    sub_d = diagnoses.loc[diagnoses[id_type].map(lambda x:x in ids), key]

    # filter for frequent diagnoses
    sub_d_value_counts = pd.DataFrame(sub_d.value_counts().head(n=50))
    sub_d_value_counts['proportion_rows'] = sub_d_value_counts[key] / len(ids)
    sub_d_value_counts.columns = ['# rows', '# rows/# IDs']

    # all diagnoses in general
    all_d = diagnoses.loc[diagnoses[key].map(lambda x:x in sub_d_value_counts.index),key]
    all_d_value_counts = pd.DataFrame(all_d.value_counts())
    all_d_value_counts['proportion_rows'] = all_d_value_counts[key] / len(ids)

    if print_output:
        print("# Codes: %s, %s. Total IDs: %i; total diagnoses: %i" % (len(codes), id_type, len(ids), len(sub_d)))
        print(value_counts)
    return ids, sub_d, value_counts

def get_coocurring_symptoms_cpt_codes(diagnoses, cpt_events, id_type, codes, print_output=True):
    assert type(codes) is list
    assert id_type in ['hadm_id', 'subject_id']
    # ids with ICD9 codes that contain any keyword in query
    ids = get_ids_with_icd_codes(diagnoses, id_type, codes)
    # all diagnoses associated with ids
    sub_d = cpt_events.loc[cpt_events[id_type].map(lambda x:x in ids), 'cpt_number']

    # filter for frequent diagnoses
    value_counts = pd.DataFrame(sub_d.value_counts().head(n=20))
    value_counts['proportion_rows'] = value_counts['cpt_number'] / len(ids)
    value_counts.columns = ['# rows', '# rows/# IDs']
    if print_output:
        print("# Codes: %s, %s. Total IDs: %i; total diagnoses: %i" % (len(codes), id_type, len(ids), len(sub_d)))
        print(value_counts)

    # TODO: fix how this function returns sth different from others
    return ids, sub_d, value_counts

def get_coocurring_symptoms_kws(diagnoses, id_type, query, print_output=True):
    assert type(query) is list
    assert id_type in ['hadm_id', 'subject_id']
    # ids with ICD9 codes that contain any keyword in query
    ids = get_ids_with_kws(diagnoses, id_type, query)
    # all diagnoses associated with ids
    sub_d = diagnoses.loc[diagnoses[id_type].map(lambda x:x in ids), 'long_title']

    # filter for frequent diagnoses
    value_counts = pd.DataFrame(sub_d.value_counts().head(n=20))

    value_counts['proportion_rows'] = value_counts['long_title'] / len(ids)
    value_counts.columns = ['# rows', '# rows/# IDs']
    if print_output:
        print("Query: %s, %s. Total IDs: %i; total diagnoses: %i" % (query, id_type, len(ids), len(sub_d)))
        print(value_counts)
    return ids, sub_d