from PyQt5.Qt import Qt, QPalette, QColor
from PyQt5.QtWidgets import QHBoxLayout, QVBoxLayout
from PyQt5.QtWidgets import QSplitter
from PyQt5.QtWidgets import QWidget

from tomcat_viz.Common.Constants import Constants
from tomcat_viz.Gui.DockEstimates import DockEstimates
from tomcat_viz.Gui.HeaderWidget import HeaderWidget
from tomcat_viz.Gui.MapWidget import MapWidget
from tomcat_viz.Gui.SpeechTranscriptionWidget import SpeechTranscriptionWidget
from tomcat_viz.Gui.ChatWidget import ChatWidget
from tomcat_viz.Gui.TimeSliderWidget import TimeSliderWidget
from tomcat_viz.Gui.Utils import createLabel
from tomcat_viz.Parser.Estimates import Estimates
from tomcat_viz.Parser.Trial import Trial

class TomcatVisualizerWidget(QWidget):
    LEFT_PANEL_PROP = 80
    MAP_HEIGHT_PROP = 80

    def __init__(self):
        super().__init__()

        self.setMinimumSize(Constants.MIN_WINDOW_SIZE)
        palette = QPalette()
        palette.setColor(QPalette.Active, QPalette.Window, QColor(Constants.Colors.APP_BACKGROUND.value))
        self.setAutoFillBackground(True)
        self.setPalette(palette)

        self._createWidgets()
        self._configureLayout()

        self._trial = None

    def loadTrialFromMetadata(self, filepath: str):
        if filepath != "":
            with open(filepath, "r") as f:
                self._trial = Trial()
                self._trial.parse(f)
            self._initializeTrial()
            return True

        return False

    def loadTrialFromPackage(self, filepath: str):
        if filepath != "":
            self._trial = Trial()
            self._trial.load(filepath)
            self._initializeTrial()
            return True

        return False

    def dumpTrial(self, filepath: str):
        if filepath != "":
            self._trial.save(filepath)

    def loadEstimates(self, filepath: str):
        if filepath != "":
            self._estimatesWidget.loadEstimates(Estimates(filepath))
            self._estimatesWidget.updateFor(self._timeSlider.value())

    def closeApp(self):
        self._estimatesWidget.close()

    def _createWidgets(self):
        self._headerPanel = HeaderWidget()

        mapWidth = int(self.width() * TomcatVisualizerWidget.LEFT_PANEL_PROP / 100)
        mapHeight = int(self.height() * TomcatVisualizerWidget.MAP_HEIGHT_PROP / 100)
        self._mapWidget = MapWidget(mapWidth, mapHeight)

        self._timeSlider = TimeSliderWidget(self._onTimeStepChange)
        self._timeSlider.setEnabled(False)

        self._chatHeaderLabel = createLabel("Chat Messages", Constants.Font.SMALL_BOLD.value, Qt.AlignLeft)
        self._chatWidget = ChatWidget()

        self._speechHeaderLabel = createLabel("Speech Transcriptions", Constants.Font.SMALL_BOLD.value, Qt.AlignLeft)
        self._speechWidget = SpeechTranscriptionWidget()

        self._estimatesHeaderLabel = createLabel("Probability Estimates", Constants.Font.SMALL_BOLD.value, "black",
                                                 Qt.AlignLeft)
        self._estimatesWidget = DockEstimates()

    def _configureLayout(self):
        mainLayout = QHBoxLayout(self)

        leftPanelLayout = QVBoxLayout()
        leftPanelLayout.addWidget(self._headerPanel, 17)
        leftPanelLayout.addWidget(self._mapWidget, TomcatVisualizerWidget.MAP_HEIGHT_PROP)
        leftPanelLayout.addWidget(self._timeSlider, 3)

        chatLayout = QVBoxLayout()
        chatLayout.setContentsMargins(0, 0, 0, 10)
        chatLayout.addWidget(self._chatHeaderLabel)
        chatLayout.addWidget(self._chatWidget)
        chatPanelWidget = QWidget()
        chatPanelWidget.setLayout(chatLayout)

        speechLayout = QVBoxLayout()
        speechLayout.setContentsMargins(0, 0, 0, 10)
        speechLayout.addWidget(self._speechHeaderLabel)
        speechLayout.addWidget(self._speechWidget)
        speechPanelWidget = QWidget()
        speechPanelWidget.setLayout(speechLayout)

        self._estimatesLayout = QVBoxLayout()
        self._estimatesLayout.setContentsMargins(0, 0, 0, 0)
        self._estimatesLayout.addWidget(self._estimatesHeaderLabel)
        self._estimatesLayout.addWidget(self._estimatesWidget.dockedWidget)
        estimatesPanelWidget = QWidget()
        estimatesPanelWidget.setLayout(self._estimatesLayout)

        rightPanelSplitter = QSplitter(Qt.Vertical)
        rightPanelSplitter.addWidget(chatPanelWidget)
        rightPanelSplitter.addWidget(speechPanelWidget)
        rightPanelSplitter.addWidget(estimatesPanelWidget)

        mainLayout.addLayout(leftPanelLayout, TomcatVisualizerWidget.LEFT_PANEL_PROP)
        mainLayout.addWidget(rightPanelSplitter, 100 - TomcatVisualizerWidget.LEFT_PANEL_PROP)
        mainLayout.setStretch(0, 0)
        mainLayout.setStretch(1, 1)

    def _onTimeStepChange(self, newTimeStep: int):
        self._updateHeaderInfo(newTimeStep)
        self._mapWidget.updateFor(newTimeStep)
        self._chatWidget.updateFor(newTimeStep)
        self._speechWidget.updateFor(newTimeStep)
        self._estimatesWidget.updateFor(newTimeStep)

    def _initializeTrial(self):
        self._mapWidget.reset()
        self._mapWidget.loadMap(self._trial.map)

        self._initializeHeaderInfo()
        self._mapWidget.loadTrial(self._trial)
        self._chatWidget.loadMessages(self._trial.chatMessages[Constants.Player.RED.value],
                                      self._trial.chatMessages[Constants.Player.GREEN.value],
                                      self._trial.chatMessages[Constants.Player.BLUE.value])
        self._speechWidget.loadMessages(self._trial.speechTranscriptions[Constants.Player.RED.value],
                                        self._trial.speechTranscriptions[Constants.Player.GREEN.value],
                                        self._trial.speechTranscriptions[Constants.Player.BLUE.value])

        self._timeSlider.setTimeSteps(self._trial.timeSteps)
        self._timeSlider.reset()
        self._timeSlider.setEnabled(True)

    def _initializeHeaderInfo(self):
        self._headerPanel.setTrialNumber(self._trial.metadata["trial_number"])
        self._headerPanel.setTeamNumber(self._trial.metadata["team_number"])
        self._headerPanel.setRedPlayerName(self._trial.metadata["red_id"])
        self._headerPanel.setGreenPlayerName(self._trial.metadata["green_id"])
        self._headerPanel.setBluePlayerName(self._trial.metadata["blue_id"])
        self._headerPanel.setRedPlayerRole(self._trial.metadata["red_role"])
        self._headerPanel.setGreenPlayerRole(self._trial.metadata["green_role"])
        self._headerPanel.setBluePlayerRole(self._trial.metadata["blue_role"])
        self._updateHeaderInfo(0)

    def _updateHeaderInfo(self, timeStep: int):
        self._headerPanel.setScore(self._trial.scores[timeStep])
        if self._trial.activeBlackout[timeStep]:
            self._headerPanel.showBlackout()
        else:
            self._headerPanel.hideBlackout()
        self._headerPanel.setRedPlayerAction(self._trial.playersActions[Constants.Player.RED.value][timeStep])
        self._headerPanel.setGreenPlayerAction(self._trial.playersActions[Constants.Player.GREEN.value][timeStep])
        self._headerPanel.setBluePlayerAction(self._trial.playersActions[Constants.Player.BLUE.value][timeStep])
        self._headerPanel.setRedPlayerEquippedItem(
            self._trial.playersEquippedItems[Constants.Player.RED.value][timeStep])
        self._headerPanel.setGreenPlayerEquippedItem(
            self._trial.playersEquippedItems[Constants.Player.GREEN.value][timeStep])
        self._headerPanel.setBluePlayerEquippedItem(
            self._trial.playersEquippedItems[Constants.Player.BLUE.value][timeStep])


from django.contrib import admin
from django.contrib.auth.models import Group
from .models import *
# Register your models here.
admin.site.site_header = "Trading Robot Admin v1.0"
admin.site.register(StockExchange)
admin.site.unregister(Group)

class StockSymbolTableAdmin(admin.ModelAdmin):
    list_display = ('stock_name', 'stock_symbols', 'get_stock_exchanges')
    fields = (
        'stock_name',
        'stock_symbols',
        'stock_exchange',
        )

admin.site.register(StockSymbolTable, StockSymbolTableAdmin)

class PlaceOrderAdmin(admin.ModelAdmin):
    list_display = ('order_id','date_time','exchange_symbol')

admin.site.register(PlaceOrder, PlaceOrderAdmin)


from optparse import OptionParser
import re
import time, datetime
import operator
import socket
from struct import unpack

# match this pattern when analyzing urls
pattern_allowed = re.compile(r'.*')

# disallow this pattern when analyzing urls
pattern_denied = re.compile(r'DISALLOWED_REGEX')

# break the log line up into named parts
parts = [
    r'(?P<host>\S+)',                   # host ip address
    r'\S+',                             # (unused)
    r'(?P<user>\S+)',                   # user
    r'\[(?P<raw_time>.+)\]',            # raw time stamp
    r'"(?P<verb>\S+)',                  # http verb
    r'(?P<request>\S+)',                # requested url
    r'(?P<http>.+)"',                   # http version
    r'(?P<status>[0-9]+)',              # status
    r'(?P<size>\S+)',                   # size (careful, can be '-')
    r'"(?P<referer>.*)"',               # referer
    r'"(?P<agent>.*)"',                 # user agent
]
pattern = re.compile(r'\s+'.join(parts)+r'\s*\Z')

# iterate through each line of the file
def readLines(input_path, output_path, verbose, stdout):
    input_file = open(input_path, 'r')
    output_file = None
    if not stdout:
        if not output_path:
            output_path = input_path + '.counter'
            if verbose:
                output_path += '.verbose'
        output_file = open(output_path, 'w')
    institution_list = get_institutions()
    raw_list = []

    # add lines to raw list
    for line in input_file:
        parsed_line = inspect(line, institution_list)
        if parsed_line:
            raw_list.append(parsed_line)

    # sort list by host and then by time
    sorted_list = sorted(raw_list, key=operator.itemgetter('host', 'time'))

    # prune the list by removing double-clicked items
    fix_double_clicks(sorted_list)

    # sort again by institution id
    sorted_list = sorted(sorted_list, key=operator.itemgetter('institution_id', 'time'))

    # print output file name
    if output_file:
        print('Output file: ' + output_path)

    # render output based on verbose flag
    if verbose:
        for item in sorted_list:
            output = str(item['institution_id']) + ' ' + item['host'] + ' ' + item['raw_time'] + ' ' + item['request']
            # send output to file or stdout
            write_output(output_file, output)
    # group items and add counts
    else:
        # list to handle counts
        grouped_list = []

        # print the headers
        header = 'institution_id year month day count'
        if output_file:
            output_file.write(header + '\n')
        else:
            print(header)

        # add items to grouped list
        for item in sorted_list:
            output = str(item['institution_id']) + ' ' + item['time'].strftime('%Y %m %d')
            group_add(grouped_list, output)

        # output grouped list
        for item in grouped_list:
            write_output(output_file, item)

# add an item to the proper group and increase the count
def group_add(grouped_list, item):
    grouped_list_length = len(grouped_list)
    group_exists = False

    # iterate through the grouped list, looking for a match
    for i in xrange(0, grouped_list_length):
        if grouped_list[i].startswith(item):
            # get the last number since that is the count
            count = int(grouped_list[i].replace(item + ' ', ''))
            # increase the count
            grouped_list[i] = item + ' ' + str(count+1)
            group_exists = True
            break

    # if the group doesn't exist, add it
    if not group_exists:
        grouped_list.append(item + ' 1')

# sends output to correct location
def write_output(output_file, line):
    # send output to file or stdout
    if output_file:
        output_file.write(line + '\n')
    else:
        print(line)

# remove items if double clicks are detected
def fix_double_clicks(sorted_list):
    sorted_list_length = len(sorted_list)
    pop_count = 0

    # for each item in the list
    for i in range(0, sorted_list_length):
        current_index = i - pop_count

        # if we are still in the bounds of the list
        if current_index < sorted_list_length - pop_count - 1:

            # if hosts and requests match, and time is within 10 seconds of the previous time, remove the line
            if sorted_list[current_index]['host'] == sorted_list[current_index+1]['host'] and sorted_list[current_index]['request'] == sorted_list[current_index+1]['request'] and sorted_list[current_index]['time'] > sorted_list[current_index+1]['time'] - datetime.timedelta(seconds=10):
                sorted_list.pop(current_index)
                pop_count += 1

        else:
            break

# create dictionary of institutions
def get_institutions():
    file = open('institution_list.txt', 'r')
    institution_list = []

    # break the institution data up into named parts
    institution_pattern = re.compile(r'(?P<institution_id>\d+)\s(?P<ip_start>\S+)\s(?P<ip_end>\S+)')

    # add lines to institution list
    for line in file:
        matches = institution_pattern.match(line)
        if matches:
            result = matches.groupdict()
            institution_list.append({'institution_id': result['institution_id'], 'ip_start': ip_encode(result['ip_start']), 'ip_end': ip_encode(result['ip_end']),})

    return institution_list

# turn the line into a dictionary
def inspect(line, institution_list):
    matches = pattern.match(line)

    # if the log line matches the apache log file pattern
    if matches:
        result = matches.groupdict()

        # if the article should be counted
        if valid_count(result):
            result['institution_id'] = check_institution(ip_encode(result['host']), institution_list)

            # if the log line is from an institution
            if result['institution_id'] > 0:
                tt = time.strptime(result['raw_time'][:-6], "%d/%b/%Y:%H:%M:%S")
                tt = list(tt[:6]) + [ 0, Timezone(result['raw_time'][-5:]) ]
                result['time'] = datetime.datetime(*tt)
                # save memory by only returning the results we need
                slim_result = {'host': result['host'], 'time': datetime.datetime(*tt), 'institution_id': result['institution_id'],
                    'request': result['request'], 'raw_time': result['raw_time']}

                # return a result if we got to here
                return slim_result

# checks if log line is from an institution
def check_institution(ip_address, institution_list):
    institution_id = 0
    # for each institution in the list
    for i in institution_list:
        # if the log line's ip address is within the institution's ip range
        if ip_address >= i['ip_start'] and ip_address <= i['ip_end']:
            institution_id = int(i['institution_id'])
            break
    return institution_id

# check if the article should be counted
def valid_count(result):
    is_counted = False

    # if a valid GET request and the specified patterns match
    if result['verb'] == 'GET' and (result['status'] == '200' or result['status'] == '304'):
        if pattern_allowed.match(result['request']):
            if not pattern_denied.search(result['request']):
                is_counted = True
    return is_counted

# convert the apache log file time to a python time object
class Timezone(datetime.tzinfo):
    def __init__(self, name="+0000"):
        self.name = name
        seconds = int(name[:-2])*3600+int(name[-2:])*60
        self.offset = datetime.timedelta(seconds=seconds)

    def utcoffset(self, dt):
        return self.offset

    def dst(self, dt):
        return None

    def tzname(self, dt):
        return self.name

# encode ip address as an unsigned long integer, useful for comparison within ranges
def ip_encode(ip_address):
    return unpack("!L", socket.inet_aton(ip_address))[0]

# run the parser
def main():
    p = OptionParser("usage: parse.py file\n\nNote: institution_list.txt (sql output of institution ip_addresses) must be present in same directory")
    p.add_option("-f", "--file", dest="output_path", help="write report to FILE", metavar="FILE")
    p.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose mode", default=False)
    p.add_option("-s", "--stdout", action="store_true", dest="stdout", help="output to standard output (terminal)", default=False)
    (options, args) = p.parse_args()

    # if a file wasn't specified
    if len(args) < 1:
        p.error("must specify a file to parse")
    output_path = options.output_path if options.output_path else None
    input_path = args[0]
    readLines(input_path, output_path, options.verbose, options.stdout)

if __name__ == '__main__':
    main()


import cv2
import numpy as np

cap = cv2.VideoCapture(0) # 0 bizim default kameramız.

#şablonu yani tamplate aranacak resimi okuduk
template = cv2.imread("python.png",0)

# şablonun boyutlarını aldık
h , w = template.shape

while True:

    ret , frame = cap.read()

    gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)

    #TM_CCOEFF_NORMED template matching algoritması kullanarak şablon işlemeyi yaptık
    res = cv2.matchTemplate(gray,template,cv2.TM_CCOEFF_NORMED)
    threshold = 0.8
    loc = np.where( res >= threshold)

    for pt in zip(*loc[::-1]):
        cv2.rectangle(frame,pt,(pt[0] + w , pt[1] + h) , (0,0,255),2)

    cv2.imshow("Webcam",frame)

    if cv2.waitKey(1) & 0xFF == ord("q") :
        break

cap.release()
cv2.destroyAllWindows()


'''Defines the command line interface for ALPALFA.'''

import argparse
from .alpafa import Lexicon
from .parse import parse_file, ParserError

def set_args():
    '''Sets command line parameters, and runs ALPAFA.'''

    parser = argparse.ArgumentParser(prog='alpafa',
                                     description='Applies the algorithm from AAFP to a correctly \
                                     formatted input file.')
    parser.add_argument('input_file', help='correctly formatted UTF-8 input file')
    parser.add_argument('output_file', help='name of file to output')

    parser.add_argument('--no_uf', dest='uf', action='store_false',
                        help='do not implement unvalued features')
    parser.add_argument('--no_cselect', dest='cselect', action='store_false',
                        help='do not implement c-selection')
    parser.add_argument('--log', dest='log', action='store_true',
                        help='include a log of algorithm operations')
    parser.add_argument('--categories', dest='cats', action='store_true',
                        help='list all categories before heads')
    parser.add_argument('--dependents', dest='dependents', action='store_true',
                        help='list all dependent features below their relevant categories (implies \
                        --categories)')

    args = parser.parse_args()
    if args.dependents:
        args.cats = True

    return (args.input_file, args.output_file, args.uf, args.cselect, args.log, args.cats,
           args.dependents)

def run_alpafa(input_file, output_file, uf, cselect, log, cats, dependents):
    '''Parse an input file, and apply ALPAFA to its contents, printing the output to a specified
    file.
    '''

    try:
        prominence, heads = parse_file(input_file)
    except FileNotFoundError as e:
        print('alpafa: input failure: ' + str(e)[10:])
        return
    except ParserError as e:
        print('alpafa: parsing failure: {}'.format(e))
        return
    lex = Lexicon(prominence, heads, uf, cselect)
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write(lex.display(log, cats, dependents))
    print(lex.stats())

def main():
    run_alpafa(*set_args())


import sys
import csv
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

def remove_duplicates(values):
    output = []
    seen = set()
    for value in values:
        # If value has not been encountered yet,
        # ... add it to both list and set.
        if value not in seen:
            output.append(value)
            seen.add(value)
    return output

#features = ['origin', 'destination', 'oil_price', 'quarter', 'year', 'flight_fare']
features = ['origin', 'destination', 'oil_price', 'flight_fare']

features2 = ['origin', 'destination', 'oil_price']
#featuresToDrop = ['origin', 'destination','distance', 'quarter', 'origin_population', 'destination_population' ,'passengers', 'flights' ,'seats', 'flight_fare']
target = ['flight_fare']
dataset = 'DS1.csv'
theData = pd.read_csv(dataset, usecols=features)

df = pd.DataFrame(theData)
#df_ = pd.DataFrame(mydata)

origin_df = df['origin']
origin_arr = np.array(np.array(origin_df))
print(origin_arr)
print(type(origin_arr))
# integer encode
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(origin_arr)
#print(integer_encoded)
# binary encode
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
print(onehot_encoded)
print(type(onehot_encoded))

with open('origin.csv', 'a') as outcsv:
    writer = csv.writer(outcsv)
    writer.writerow(["Origin", "Origin_Encoded"])

with open('destination.csv', 'a') as outcsv:
    writer = csv.writer(outcsv)
    writer.writerow(["Destination", "Destination_Encoded"])

origin_encoded_list = []
for origin_abbr, origin_encoded in zip(origin_arr,onehot_encoded):
    #print("{} --> {}".format(origin_abbr,origin_encoded))
    #origin_encoded_list.append([origin_abbr,origin_encoded])
    #origin_encoded = origin_encoded.replace('\n','')
    with open('origin.csv', 'a') as file:
        writer = csv.writer(file, delimiter = ",")
        writer.writerow([origin_abbr,origin_encoded])

#===================================

destination_df = df['destination']
destination_arr = np.array(remove_duplicates(np.array(destination_df)))
print(destination_arr)
print(type(destination_arr))
# integer encode
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(destination_arr)
#print(integer_encoded)
# binary encode
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
print(onehot_encoded)
print(type(onehot_encoded))

origin_encoded_list = []
for origin_abbr, origin_encoded in zip(destination_arr,onehot_encoded):
    #print("{} --> {}".format(origin_abbr,origin_encoded))
    #origin_encoded_list.append([origin_abbr,origin_encoded])
    with open('destination.csv', 'a') as file:
        writer = csv.writer(file, delimiter = ",")
        writer.writerow([origin_abbr,origin_encoded])

#print(origin_encoded_list)
#print(df)


import os
import json
import boto3
from boto3.session import Session
#import openpyxl
from botocore.exceptions import ClientError
import datetime
import logging
#import pytz
#import tzlocal
import dateutil.parser
from pprint import pprint
###
#--- [ref] change region programmatically
#https://russell.ballestrini.net/setting-region-programmatically-in-boto3/

#????

#--- RESOURCES AND CLIENTS ---------
#====================================
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3Resource = boto3.resource('s3')
#bclient = boto3.client('ce')
athena_client = boto3.client('athena')
budgets_client = boto3.client('budgets')
dynamodb = boto3.resource('dynamodb')
ec2_client = boto3.client('ec2')
rsclient = boto3.client('redshift')
iamc = boto3.client('iam')

reg_arr = []
count = 0

#=====================================
"""
https://www.decalage.info/en/python/print_list

"""

#aws_acct_num = boto3.client('sts').get_caller_identity()['Account']

#-- EVENTS RULE REFERENCE
"""https://github.com/lroguet/amzn-cloudformation/blob/master/lambda/scheduled-lambda.yml
"""

#-- ATTACH IAM POLICY TO A ROLE
"""
response = client.attach_role_policy(
    RoleName='string',
    PolicyArn='string'
)

"""

#--- LIST ALL IAM USERS FOR AN ACCOUNT
"""
for user in iamc.list_users()['Users']:
  user_name = user['UserName']
  print(user_name)
"""
#--- PRINT ALL IAM ACCESS KEYS FOR USERS WHO HAVE ONE OR MORE
"""
  keys = iamc.list_access_keys(UserName=user)
  #print(keys)
  for b in keys['AccessKeyMetadata']:
      username = b['UserName']
      accesskeyid = b['AccessKeyId']
      status = b['Status']
      createdate = str(b['CreateDate']) # creation date
      print("{0:<20} {1:<25} {2:<15} {3:<10}".format(username, accesskeyid, status, createdate))
      #access_key = iam.AccessKey(username,accesskeyid)
        #access_key.deactivate()
"""

#--- PRINT ALL SSH KEYS FOR USERS WHO HAVE ONE OR MORE
"""
iamc = boto3.client('iam')
iamr = boto3.resource('iam')
x=0
for user in iamc.list_users()['Users']:

  user_name = user['UserName']
 #['SSHPublicKeys']
 #['ResponseMetaData']
  try:
    response = iamc.list_ssh_public_keys(UserName=user_name)
    for item in response.values():
      #print(user_name) # prints all users
      num_items = len(item)
      if num_items >= 1:
        for each in item:
          print(each['SSHPublicKeyId']) # prints users  and the keys of thosewho have ssh keys
    x = x + 1
  except:
    continue
"""
#--- REPORT CLASS
"""
class Report:
  def __init__(aws_acct_num, creation_date,rsrc_type,rsrc_name,owner):
    self.aws_acct_num = aws_acct_num
    slef.reation_date = reation_date
    self.rsrc_type = rsrc_type
    self.rsrc_name = rsrc_name
    self.owner = owner

  def write_report(aws_acct_num, creation_date,rsrc_type,rsrc_name,owner):
    wb = openpyxl.load_workbook('inventory.xlsx')
    wb.sheetnames
    sheet = wb.active
    currSheet = sheet.title

    sheet.append((aws_acct_num,readable_date,"s3",bucket['Name'], bucket_owner))
    print("mark")
    wb.save('inventory.xlsx')
"""
#--- CREATE CLOUDWATCH CLIENT
"""
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]

for region in regions:
  print(region)
  bclient = boto3.client('redshift', region_name = region)
  cluster_list = response = bclient.describe_cluster_parameters(
  ParameterGroupName = 'default',
  )
  for cluster in cluster_list:
    pprint(cluster)
"""

#--- GET AWS ACCOUNT NUMBER
"""
boto3.client('sts').get_caller_identity()['Account']
def lambda_handler(event, context):
  print("My account ID is %s" % get_account_id(context))

"""

#--- COMPARING DATES IN PYTHON

time=[]
dt = datetime.datetime.now()
time1 = datetime.datetime(2019, 11, 1)
time2 = datetime.datetime(2020, 3, 30)
time.append(time1)
time.append(time2)
print(time1)
print(time2)
print(time2 - (datetime.timedelta(30)))
past_thirty = time2 - (datetime.timedelta(30))

for x in time:

  if(x <= time2 and x >= past_thirty):
    print(str(x) + " this time is in range")
  else:
    print(str(x) + " this time not in range")

#--- RETRIEVE S3 BUCKET POLICY
"""
result = s3.get_bucket_policy(Bucket='74481-wfm-eidin-flq-explore-sbx-logging')
print(result['Policy'])
"""
#--- CLOUDFORMATION
"""
cfn.list_stacks()
print(type(cfn))
"""

#--- LIST ALL STACKS IN AN ACCOUNT
"""
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
count = 1
for region in regions:
  reg_arr.append(region)
  cfn = boto3.client('cloudformation', region_name = region)
  for stack in cfn.list_stacks(
    StackStatusFilter = [
    'CREATE_COMPLETE', 'UPDATE_COMPLETE'
    ])['StackSummaries']:
      #print('{0}:{1}'.format(stack['StackName'], stack['StackId']))
      count += 1
      print(stack['StackName'])
      #print(stack['StackId'])
      continue
print("here's the count: " + str(count))

#for reg in reg_arr:
  #print(reg)

"""

#--- LIST ALL REGIONS
"""
ec2_client = boto3.client('ec2')
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
for region in regions:
  reg_arr.append(region)

for region in reg_arr:
  print(region)
#regions = ec2_client.describe_regions().get('Regions',[] )
#print(reg_arr)
"""

#--- PRINT ALL REGIONS THAT ARE ACTIVE
"""
for region in regions:
  reg_arr.append(region)
  continue
"""

#--- PRINT THE NUMBER OF BUCKETS
"""
print("lot's of buckets here guys:"+str(len(response['Buckets'])))
"""

#--- CREATE AN S3 BUCKET POLICY
"""
bucket_name = 'BUCKET_NAME'
bucket_policy = {
    'Version': '2012-10-17',
    'Statement': [{
        'Sid': 'AddPerm',
        'Effect': 'Allow',
        'Principal': '*',
        'Action': ['s3:GetObject'],
        'Resource': f'arn:aws:s3:::{bucket_name}/*'
    }]
}

#--- Convert the policy from JSON dict to string
bucket_policy = json.dumps(bucket_policy)

# Set the new policy
s3 = boto3.client('s3')
s3.put_bucket_policy(Bucket=bucket_name, Policy=bucket_policy)

"""

#--- GET S3 BUCKET TAGS
"""
s3 = boto3.client('s3')
buckets = s3_client.list_buckets()
s3r = boto3.resource('s3')

response_c = s3.list_buckets()
buckets = response_c
for bucket in response_c['Buckets']:
  try:
    #print(bucket['Name'])
    raw_date = bucket['CreationDate']
    readable_date = raw_date.strftime("%m-%d-%Y")
    time_today = datetime.datetime.now()
    past_thirty = time_today - (datetime.timedelta(30))
    bucket_tag = s3.get_bucket_tagging(Bucket = bucket['Name'])
    bucket_owner_tag = bucket_tag['TagSet'][0]['Key']
    if(bucket_owner_tag == 'owner'):
      print(bucket_tag['TagSet'][0]['Value'])
    else:
      continue
  except:
    continue
  #print("Bucket Name: " + bucket['Name']+ " -- Creation Date: " + readable_date)

[COMMENT]
for bucket in buckets:
  try:
      print(bucket)
      response = s3_client.get_bucket_tagging(Bucket=bucket)
      print(response['owner'])
      #tagset = response['TagSet']
  except ClientError:
      print(bucket, "does not have tags, adding tags")
[END_COMMENT]

"""

#--- S3 - LIST BUCKET NAMES AND CREATION DATES -- https://www.w3schools.com/python/python_datetime.asp
"""

s3 = boto3.client('s3')
buckets = s3_client.list_buckets()
s3r = boto3.resource('s3')
aws_acct_num = boto3.client('sts').get_caller_identity()['Account']
wb = openpyxl.load_workbook('inventory.xlsx')
wb.sheetnames
sheet = wb.active
currSheet = sheet.title
response_c = s3_client.list_buckets()
buckets = response_c
for bucket in response_c['Buckets']:
  try:
    raw_date = bucket['CreationDate']
    readable_date = raw_date.strftime("%m-%d-%Y")
    time_today = datetime.datetime.now()
    past_thirty = time_today - (datetime.timedelta(30))
    bucket_tag = s3.get_bucket_tagging(Bucket = bucket['Name'])
    bucket_owner_tag = bucket_tag['TagSet'][0]['Key']

    if(raw_date.date() >= past_thirty.date()):
      bucket_tagging = s3r.get_bucket_tagging(Bucket = bucket['Name'])
      bucket_owner_tag = bucket_tag['TagSet'][0]['Key']
      #print(bucket['Name'])
      print(bucket_owner_tag)
    if(bucket_owner_tag == 'owner'):
      bucket_owner = bucket_tag['TagSet'][0]['Value']
      print(bucket_owner)
    else:
      bucket_owner = "no tag found"
      continue

    sheet.append((aws_acct_num,readable_date,"s3",bucket['Name'], bucket_owner))
    #print(bucket)
    #sheet.append(B,[readable_date])
    #print(raw_date.strftime("%x"))
    #print(bucket)
    #print(bucket['CreationDate'])

    wb.save('inventory.xlsx')
  except:
    continue
"""

#--- S3 LOGGING INFO
"""
bucket_logging = s3Resource.BucketLogging('74481-jeffs-test-land')
print(bucket_logging)
"""
#--- s3 ACL
"""
s3ACL = s3.get_bucket_acl('2120950-land')
s3ACL = s3.acl('2120950-land')
print(s3ACL)

"""
#--- OPEN OPENPYXL
""""
wb = openpyxl.load_workbook('inventory.xlsx')
wb.sheetnames
sheet = wb.active
currSheet = sheet.title

names=[]

for bucket in response['Buckets']:
  try:
    bucket_tagging = s3.get_bucket_tagging(Bucket = bucket['Name'])
    #print(bucket['Name'])
    #print(bucket_tagging)
    sheet.append([bucket['Name']])

    #print(get_s3_keys(bucket['Name']))
  except:
    continue
    #print("XXX")

names.append(names)
wb.save(filename='/mnt/c/Users/jjone/Downloads/sbx_python_boto/inventory.xlsx')
#wb.save()
"""

"""
print(type(sheet))

for row in sheet.values:
  for value in row:
    print(value)

print(currSheet)
"""

"""
x = 1
#print(f'  {bucket["Name"]}')

workbook = Workbook()
sheet = workbook.active
for bucket in response['Buckets']:

  print(sheet["A"+str(x)])

  x+=x

workbook.save(filename="./inventory.xlsx")
LANDING_BUCKET_NAME = os.environ['LANDING_BUCKET_NAME']
PERSIST_BUCKET_NAME = os.environ['PERSIST_BUCKET_NAME']
SNS = os.environ['SNS']

this will get all the regions that dynamoDB is available in

s = Session()
dynamodb_regions = s.get_available_regions('dynamodb')
for dynamo_region in dynamodb_regions:
  print(dynamo_region)

"""

#---- GET A LIST OF ALL TOPIC ARNS FROM THE RESPONSE
"""
topics = [topic['TopicArn'] for topic in response['Topics']]

 Print out the topic list
print("Topic List: %s" % topics)
snstopic = 'arn:aws:sns:us-east-1:744810512381:NotifyJeff'
if snstopic in topics:
    #print("we found your topic")
"""

"""
    # log event and context attributes
    print(json.dumps(event, default=str, indent=4))
    print(json.dumps(context.__dict__, default=str, indent=4))

    # get bucket name and object key attributes from event object

    source_bucket_name = event['Records'][0]['s3']['bucket']['name']

    source_object_key = urllib.parse.unquote(event['Records'][0]['s3']['object']['key'])
    print(f"{source_object_key} PUT to {source_bucket_name}")
"""
"""
    f = open("/tmp/jeffstest.txt", "a")
    for x in event:
        print(x)
        f.write(x)
    f.close()
    s3.Object('jeffs-test-land-copy1', 'jeffstest.txt').put(Body=open('/tmp/jeffstest.txt', 'rb'))
    #s3.upload_fileobj(f, "jeffs-test-land-main", "somefile")
"""
"""
    if not source_bucket_name == LANDING_BUCKET_NAME:
        raise EnvironmentError("event source_bucket_name NOT equals environment LANDING_BUCKET_NAME")

    # build copy source and target object instance from event object attributes
    copy_source = {'Bucket': source_bucket_name, 'Key': source_object_key}
    target_obj = s3.Object(PERSIST_BUCKET_NAME, source_object_key)

    print(f"{source_object_key} COPY to {PERSIST_BUCKET_NAME}")
    target_obj.copy_from(CopySource=copy_source)

    print(f"wait for {source_object_key} to exist in {PERSIST_BUCKET_NAME}")
    target_obj.wait_until_exists()

    print(f"{source_object_key} COPY to {PERSIST_BUCKET_NAME} successful")
"""
#--- scrap
"""
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3r = s3_client.list_buckets()
buckets = s3r
for bucket in s3r['Buckets']:

  bucket_tagging = s3r.BucketTagging(Bucket = bucket['Name'])
  #print(bucket['Name'])
  print(bucket_tagging)
  #sheet.append([bucket['Name']])

    #print(get_s3_keys(bucket['Name']))
"""


import h5py
import json
import pandas as pd

import cellranger.analysis.singlegenome as cr_sg_analysis
import cellranger.analysis.multigenome as cr_mg_analysis
import cellranger.webshim.constants.gex as gex_constants

class SampleDataPaths:
    def __init__(self, summary_path=None, barcode_summary_path=None, analysis_path=None,
                 vdj_clonotype_summary_path=None,
                 vdj_barcode_support_path=None):
        self.summary_path = summary_path
        self.barcode_summary_path = barcode_summary_path
        self.analysis_path = analysis_path
        self.vdj_clonotype_summary_path = vdj_clonotype_summary_path
        self.vdj_barcode_support_path = vdj_barcode_support_path

class SampleData:
    def __init__(self, sample_properties, sample_data_paths, plot_preprocess_func):
        self.summary = load_metrics_summary(sample_data_paths.summary_path)

        # Not guaranteed to exist for all pipelines
        self.num_cells = self.summary.get('filtered_bcs_transcriptome_union')

        self.analysis, self.original_cluster_sizes = load_analysis(sample_data_paths.analysis_path,
                                                                   plot_preprocess_func,
                                                                   sample_properties)

        self.barcode_summary = load_barcode_summary(sample_data_paths.barcode_summary_path)

        self.vdj_clonotype_summary = pd.read_csv(sample_data_paths.vdj_clonotype_summary_path) if sample_data_paths.vdj_clonotype_summary_path else None

        self.vdj_barcode_support = pd.read_csv(sample_data_paths.vdj_barcode_support_path, usecols=[1]) if sample_data_paths.vdj_barcode_support_path else None

def load_metrics_summary(summary_path):
    if summary_path is None:
        return None
    with open(summary_path) as f:
        return json.load(f)

def load_analysis(base_dir, plot_preprocess_func, sample_properties):
    """ Returns (analysis_object, original_cluster_sizes) """
    if base_dir is None:
        return None, None

    if len(sample_properties['genomes']) == 1:
        analysis = cr_sg_analysis.SingleGenomeAnalysis.load_default_format(base_dir)
    else:
        analysis = cr_mg_analysis.MultiGenomeAnalysis.load_default_format(base_dir)

    if analysis is None:
        return None, None

    # Subsample barcodes for plotting purposes
    original_cluster_sizes = None
    if analysis is not None and isinstance(analysis, cr_sg_analysis.SingleGenomeAnalysis):
        original_cluster_sizes = analysis.get_cluster_sizes()
        if analysis.matrix.bcs_dim > gex_constants.MAX_WEBSHIM_BCS_DIM:
            analysis.subsample_bcs(gex_constants.MAX_WEBSHIM_BCS_DIM)

    return plot_preprocess_func(analysis, sample_properties), original_cluster_sizes

def load_barcode_summary(h5_path):
    if h5_path is None:
        return None
    return h5py.File(h5_path, 'r')


from collections import OrderedDict, defaultdict
from dataclasses import dataclass
import os
import json
from typing import Dict, Iterable, List, Optional, Set, Tuple

import dacite

from helm.benchmark.adaptation.adapters.adapter_factory import (
    ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS,
    ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
)
from helm.benchmark.adaptation.adapter_spec import AdapterSpec
from helm.benchmark.adaptation.request_state import RequestState
from helm.benchmark.adaptation.scenario_state import ScenarioState
from helm.benchmark.augmentations.dialect_perturbation import DialectPerturbation
from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation
from helm.benchmark.augmentations.filler_words_perturbation import FillerWordsPerturbation
from helm.benchmark.augmentations.gender_perturbation import GenderPerturbation
from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
from helm.benchmark.augmentations.person_name_perturbation import PersonNamePerturbation
from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
from helm.benchmark.augmentations.synonym_perturbation import SynonymPerturbation
from helm.benchmark.augmentations.typos_perturbation import TyposPerturbation
from helm.benchmark.metrics.metric import PerInstanceStats
from helm.benchmark.presentation.schema import Schema
from helm.benchmark.runner import RunSpec
from helm.benchmark.scenarios.scenario import Instance
from helm.common.general import asdict_without_nones, write
from helm.common.hierarchical_logger import htrack
from helm.common.request import Request

# TODO(#1251): Add proper class registration
_PERTURBATION_NAME_TO_DESCRIPTION = {
    DialectPerturbation.name: DialectPerturbation.Description,
    ExtraSpacePerturbation.name: ExtraSpacePerturbation.Description,
    FillerWordsPerturbation.name: FillerWordsPerturbation.Description,
    GenderPerturbation.name: GenderPerturbation.Description,
    MisspellingPerturbation.name: MisspellingPerturbation.Description,
    PersonNamePerturbation.name: PersonNamePerturbation.Description,
    SpacePerturbation.name: SpacePerturbation.Description,
    SynonymPerturbation.name: SynonymPerturbation.Description,
    TyposPerturbation.name: TyposPerturbation.Description,
}

def _deserialize_perturbation_description(raw_perturbation_description: Dict) -> PerturbationDescription:
    """Convert a raw dictionary to a PerturbationDescription.
    This uses the name field to look up the correct PerturbationDescription subclass to output.
    """
    factory = _PERTURBATION_NAME_TO_DESCRIPTION.get(raw_perturbation_description["name"], PerturbationDescription)
    return factory(**raw_perturbation_description)

_DACITE_CONFIG = dacite.Config(type_hooks={PerturbationDescription: _deserialize_perturbation_description})

@dataclass(frozen=True)
class DisplayPrediction:
    """
    Captures a unit of evaluation for displaying in the web frontend.
    """

    # (instance_id, perturbation, train_trial_index) is a unique key for this prediction.
    instance_id: str
    """ID of the Instance"""

    perturbation: Optional[PerturbationDescription]
    """Description of the Perturbation that was applied"""

    train_trial_index: int
    """Which replication"""

    predicted_text: str
    """Prediction text"""

    truncated_predicted_text: Optional[str]
    """The truncated prediction text, if truncation is required by the Adapter method."""

    mapped_output: Optional[str]
    """The mapped output, if an output mapping exists and the prediction can be mapped"""

    reference_index: Optional[int]
    """Which reference of the instance we're evaluating (if any)"""

    stats: Dict[str, float]
    """Statistics computed from the predicted output"""

@dataclass(frozen=True)
class DisplayRequest:
    """
    Captures a unit of evaluation for displaying in the web frontend.
    """

    # (instance_id, perturbation, train_trial_index) is a unique key for this prediction.
    instance_id: str
    """ID of the Instance"""

    perturbation: Optional[PerturbationDescription]
    """Description of the Perturbation that was applied"""

    train_trial_index: int
    """Which replication"""

    request: Request
    """The actual Request to display in the web frontend.

    There can be multiple requests per trial. The displayed request should be the
    most relevant request e.g. the request for the chosen cohice for multiple choice questions."""

def _read_scenario_state(run_path: str) -> ScenarioState:
    scenario_state_path: str = os.path.join(run_path, "scenario_state.json")
    if not os.path.exists(scenario_state_path):
        raise ValueError(f"Could not load ScenarioState from {scenario_state_path}")
    with open(scenario_state_path) as f:
        raw_scenario_state = json.load(f)
        return dacite.from_dict(ScenarioState, raw_scenario_state, config=_DACITE_CONFIG)

def _read_per_instance_stats(run_path: str) -> List[PerInstanceStats]:
    per_instance_stats_path: str = os.path.join(run_path, "per_instance_stats.json")
    if not os.path.exists(per_instance_stats_path):
        raise ValueError(f"Could not load PerInstanceStats from {per_instance_stats_path}")
    with open(per_instance_stats_path) as f:
        raw_per_instance_stats = json.load(f)
        return [dacite.from_dict(PerInstanceStats, r, config=_DACITE_CONFIG) for r in raw_per_instance_stats]

def _truncate_predicted_text(
    predicted_text: str, request_state: RequestState, adapter_spec: AdapterSpec
) -> Optional[str]:
    method = adapter_spec.method
    prefix = ""
    if method in ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS:
        prefix = request_state.instance.input.text
    elif method == "language_modeling":
        if request_state.result is not None and request_state.result.completions:
            tokens = request_state.result.completions[0].tokens
            if tokens:
                first_token = tokens[0]
                if not first_token.top_logprobs:
                    prefix = first_token.text
    if prefix:
        predicted_text = predicted_text
        prefix = prefix
        if predicted_text.startswith(prefix):
            return predicted_text[len(prefix) :]
    return None

def _get_metric_names_for_group(run_group_name: str, schema: Schema) -> Set[str]:
    metric_groups_by_name = {metric_group.name: metric_group for metric_group in schema.metric_groups}
    run_groups_by_name = {run_group.name: run_group for run_group in schema.run_groups}

    result: Set[str] = set()
    run_group = run_groups_by_name.get(run_group_name)
    if run_group is None:
        return result

    for metric_group_name in run_group.metric_groups:
        metric_group = metric_groups_by_name.get(metric_group_name)
        if metric_group is None:
            continue
        for metric_name_matcher in metric_group.metrics:
            if metric_name_matcher.perturbation_name:
                continue
            result.add(metric_name_matcher.substitute(run_group.environment).name)
    return result

def _get_metric_names_for_groups(run_group_names: Iterable[str], schema: Schema) -> Set[str]:
    result: Set[str] = set()
    for run_group_name in run_group_names:
        result.update(_get_metric_names_for_group(run_group_name, schema))
    return result

@htrack(None)
def write_run_display_json(run_path: str, run_spec: RunSpec, schema: Schema):
    """Write run JSON files that are used by the web frontend.

    The derived JSON files that are used by the web frontend are much more compact than
    the source JSON files. This speeds up web frontend loading significantly.

    Reads:

    - ScenarioState from `scenario_state.json`
    - List[PerInstanceStats] from `per_instance_stats.json`

    Writes:

    - List[Instance] to `instances.json`
    - List[DisplayPrediction] to `display_predictions.json`
    - List[DisplayRequest] to `display_requests.json`
    """
    scenario_state = _read_scenario_state(run_path)
    per_instance_stats = _read_per_instance_stats(run_path)

    metric_names = _get_metric_names_for_groups(run_spec.groups, schema)

    if run_spec.adapter_spec.method in ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS:
        metric_names.add("predicted_index")

    stats_by_trial: Dict[Tuple[str, Optional[PerturbationDescription], int], Dict[str, float]] = defaultdict(dict)
    for original_stats in per_instance_stats:
        stats_dict: Dict[str, float] = {
            original_stat.name.name: original_stat.mean
            for original_stat in original_stats.stats
            if original_stat.name.name in metric_names and original_stat.mean is not None
        }

        key = (
            original_stats.instance_id,
            original_stats.perturbation,
            original_stats.train_trial_index,
        )
        stats_by_trial[key].update(stats_dict)

    instance_id_to_instance: Dict[Tuple[str, Optional[PerturbationDescription]], Instance] = OrderedDict()
    predictions: List[DisplayPrediction] = []
    requests: List[DisplayRequest] = []

    for request_state in scenario_state.request_states:
        assert request_state.instance.id is not None
        if request_state.result is None:
            continue

        # For the multiple_choice_separate_calibrated adapter method,
        # only keep the original prediction and discard the calibration prediction.
        if (
            run_spec.adapter_spec.method == ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED
            and request_state.request_mode == "calibration"
        ):
            continue

        stats_key = (
            request_state.instance.id,
            request_state.instance.perturbation,
            request_state.train_trial_index,
        )
        trial_stats: Dict[str, float] = stats_by_trial[stats_key]
        # For the multiple_choice_separate_* adapter methods,
        # only keep the prediction for the chosen reference and discard the rest.
        if (
            run_spec.adapter_spec.method in ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS
            and "predicted_index" in trial_stats
            and trial_stats["predicted_index"] != request_state.reference_index
        ):
            continue

        predicted_text = (
            request_state.result.completions[0].text
            if request_state.result is not None and request_state.result.completions
            else ""
        )
        mapped_output = (
            request_state.output_mapping.get(predicted_text.strip()) if request_state.output_mapping else None
        )
        instance_id_to_instance[
            (request_state.instance.id, request_state.instance.perturbation)
        ] = request_state.instance
        predictions.append(
            DisplayPrediction(
                instance_id=request_state.instance.id,
                perturbation=request_state.instance.perturbation,
                train_trial_index=request_state.train_trial_index,
                predicted_text=predicted_text,
                truncated_predicted_text=_truncate_predicted_text(predicted_text, request_state, run_spec.adapter_spec),
                mapped_output=mapped_output,
                reference_index=request_state.reference_index,
                stats=trial_stats,
            )
        )
        requests.append(
            DisplayRequest(
                instance_id=request_state.instance.id,
                perturbation=request_state.instance.perturbation,
                train_trial_index=request_state.train_trial_index,
                request=request_state.request,
            )
        )

    write(
        os.path.join(run_path, "instances.json"),
        json.dumps(list(map(asdict_without_nones, instance_id_to_instance.values())), indent=2),
    )
    write(
        os.path.join(run_path, "display_predictions.json"),
        json.dumps(list(map(asdict_without_nones, predictions)), indent=2),
    )
    write(
        os.path.join(run_path, "display_requests.json"),
        json.dumps(list(map(asdict_without_nones, requests)), indent=2),
    )


import cv2
import numpy as np
import sys

if __name__ == '__main__':
    array = np.array([[1, 1, 1, 1, 1],
                      [1, 1, 1, 1, 1],
                      [1, 1, 0, 1, 1],
                      [1, 1, 1, 1, 1],
                      [1, 1, 1, 1, 1]], np.uint8)
    # 分别计算街区距离、欧式距离和棋盘距离
    dst_L1 = cv2.distanceTransform(array, cv2.DIST_L1, cv2.DIST_MASK_3)
    dst_L2 = cv2.distanceTransform(array, cv2.DIST_L2, cv2.DIST_MASK_5)
    dst_C = cv2.distanceTransform(array, cv2.DIST_C, cv2.DIST_MASK_3)

    # 对图像进行读取
    rice = cv2.imread('rice.png', cv2.IMREAD_GRAYSCALE)
    if rice is None:
        print('Failed to read img')
        sys.exit()
    # 将图像转化为二值图像，同时将黑白区域颠倒
    rice_BW = cv2.threshold(rice, 50, 255, cv2.THRESH_BINARY)
    rice_BW_INV = cv2.threshold(rice, 50, 255, cv2.THRESH_BINARY_INV)

    # 图像距离变换
    dst_rice_BW = cv2.distanceTransform(rice_BW[1], 1, 3, dstType=cv2.CV_32F)
    dst_rice_INV = cv2.distanceTransform(rice_BW_INV[1], 1, 3, dstType=cv2.CV_8U)
    print(dst_rice_INV)

    print('街区距离：\n{}'.format(dst_L1))
    print('欧式距离：\n{}'.format(dst_L2))
    print('棋盘距离：\n{}'.format(dst_C))

    cv2.imshow('rice_bw', rice_BW[1])
    cv2.imshow('rice_bw_INV', rice_BW_INV[1])
    cv2.imshow('dst_rice_bw', dst_rice_BW)
    cv2.imshow('dst_rice_bw_inv', dst_rice_INV)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

6. 图像形态学操作/6.1.1 图像距离变换.py


import shutil
import warnings
from typing import Tuple
import numpy as np
import cv2

from PIL import Image
from config import *
from img_proc_util import *
##  sklearn image related functions

import skimage.color
import skimage.io
import skimage.metrics
from skimage import img_as_ubyte
## importing torch and torchvision utilities
import torch
import torchvision.utils

def image_quality_assessment(sr_path: str, hr_path: str) -> Tuple[float, float, float]:

    ## reading hr and sr image
    hr_image = skimage.io.imread(hr_path)
    sr_image = skimage.io.imread(sr_path)

    if sr_image.shape[1] != sr_image.shape[0]:
        warnings.warn("Image shape - height and width not equal")
    if hr_image.shape != sr_image.shape:
        warnings.warn("spectrum calculation error")

    spectrum = cal_spectrum(sr_image, hr_image)
    ## calculation of PSNR and SSIM
    psnr, ssim = cal_psnr_and_ssim(sr_image, hr_image)

    return psnr, ssim, spectrum

def cal_spectrum(sr_image, hr_image) -> float:

    sr_temp = img_as_ubyte(sr_image)
    hr_temp = img_as_ubyte(hr_image)
    sr = cv2.cvtColor(sr_temp, cv2.COLOR_RGB2GRAY)
    hr = cv2.cvtColor(hr_temp, cv2.COLOR_RGB2GRAY)

    sr_shape = sr.shape[0]
    total_hist_sr = []
    total_hist_hr = []
    for hist_height in range(sr_shape):

        hist_sr = cv2.calcHist([sr[hist_height, :]], [0], None, [sr_shape], [0, 255])
        hist_hr = cv2.calcHist([hr[hist_height, :]], [0], None, [sr_shape], [0, 255])
        total_hist_sr.append(hist_sr)
        total_hist_hr.append(hist_hr)

    total_spectrum_sr = []
    total_spectrum_hr = []
    for idx in range(sr_shape):

        fft_sr = np.fft.fft(total_hist_sr[idx])
        fft_hr = np.fft.fft(total_hist_hr[idx])

        spectrum_sr = np.abs(fft_sr)
        spectrum_hr = np.abs(fft_hr)

        spectrum_sr = spectrum_sr[range(sr_shape // 2)]
        spectrum_hr = spectrum_hr[range(sr_shape // 2)]
        total_spectrum_sr.append(spectrum_sr)
        total_spectrum_hr.append(spectrum_hr)

    average_spectrum_sr = []
    average_spectrum_hr = []
    diff = 0.
    for spectrum_val in range(sr_shape // 2):
        total_spectrum_sr = 0
        total_spectrum_hr = 0
        for idx in range(sr_shape):
            total_spectrum_sr += total_spectrum_sr[idx][spectrum_val]
            total_spectrum_hr += total_spectrum_hr[idx][spectrum_val]
        average_spectrum_sr.append(total_spectrum_sr / sr_shape)
        average_spectrum_hr.append(total_spectrum_hr / sr_shape)

    for index in range(sr_shape // 2):
        diff += (average_spectrum_hr[index] - average_spectrum_sr[index]) ** 2

    spectrum_val = float(np.sqrt(diff / (sr_shape / 2)))

    return spectrum_val

def cal_psnr_and_ssim(sr_image, hr_image) -> Tuple[float, float]:
    ## calculate psnr and ssim

    sr_temp = normalize(sr_image)
    hr_temp = normalize(hr_image)

    sr_temp = skimage.color.rgb2ycbcr(sr_temp)[:, :, 0:1]
    hr_temp = skimage.color.rgb2ycbcr(hr_temp)[:, :, 0:1]

    sr = normalize(sr_temp)
    hr = normalize(hr_temp)
    ssim = skimage.metrics.structural_similarity(sr,
                                                 hr,
                                                 win_size=11,
                                                 gaussian_weights=True,
                                                 multichannel=True,
                                                 data_range=1.0,
                                                 K1=0.01,
                                                 K2=0.03,
                                                 sigma=1.5)
    psnr = skimage.metrics.peak_signal_noise_ratio(sr, hr, data_range=1.0)

    return psnr, ssim

def main() -> None:
    ## defining the main function to get the functions in line

    if os.path.exists(exp_dir):
        shutil.rmtree(exp_dir)
    os.makedirs(exp_dir)

    ## load models to evaluate
    state_dict = torch.load(model_path, map_location=device)
    model.load_state_dict(state_dict)

    model.eval()

    model.half()

    total_spectrum_val = 0.0
    total_ssim_val = 0.0
    total_psnr_val = 0.0

    filenames = os.listdir(lr_dir)

    total_files = len(filenames)

    for index in range(total_files):
        lr_path = os.path.join(lr_dir, filenames[index])
        sr_path = os.path.join(sr_dir, filenames[index])
        hr_path = os.path.join(hr_dir, filenames[index])

        lr_image = Image.open(lr_path)
        lr_tensor = image2tensor(lr_image).to(device).unsqueeze(0)
        lr_tensor = lr_tensor.half()
        with torch.no_grad():
            sr_tensor = model(lr_tensor)
            torchvision.utils.save_image(sr_tensor, sr_path)

        print(f"Processing `{os.path.abspath(lr_path)}`...")
        psnr, ssim, spectrum = image_quality_assessment(sr_path, hr_path)
        total_spectrum_val += spectrum

        total_ssim_val += ssim
        total_psnr_val += psnr

    print(f"PSNR:     {total_psnr_val / total_files:.2f}.\n"
          f"SSIM:     {total_ssim_val / total_files:.4f}.\n"
          f"Spectrum: {total_spectrum_val / total_files:.6f}.\n")

if __name__ == "__main__":
    main()


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.1
#   kernelspec:
#     display_name: causal-covid-analysis
#     language: python
#     name: causal-covid-analysis
# ---

# # Severe Covid Cases and Vaccinations in Israel

# +
import os

import pandas as pd
import datetime

from pathlib import Path

# +
# data paths
INPUT_DIR = Path("../data/input-data")
HOSP_DATA_PATH = INPUT_DIR / 'event-among-vaccinated-120.csv'
VACCINATION_DATA_PATH = INPUT_DIR / 'vaccinated-per-day-2022-01-01.csv'
CASE_DATA_PATH = INPUT_DIR / 'cases-among-vaccinated-185.csv'
UN_POPULATION_DATA_PATH = INPUT_DIR / 'WPP2019_POP_F07_1_POPULATION_BY_AGE_BOTH_SEXES.csv'

OUTPUT_DIR = Path("../data/preprocessed-data")
OUTPUT_DIR.mkdir(exist_ok=True)
# -

# # 2 Dataset: Fine Grained Age Groups

# ## 2.1 Load Data

# ### 2.1.1 Deaths and Hospitalisations

# Remark on the vaccination status in this table: people are counted as vaccinated from the day after the dose is given. This means there is no 14 day period between dose and updated vaccination status (see event-among-vaccinated-81-readme.pdf).

df_tmp = pd.read_csv(HOSP_DATA_PATH)
df_tmp['Sunday_date'] = pd.to_datetime(df_tmp['Week'].str[:10], format='%Y-%m-%d')
df_tmp = df_tmp.drop(columns=['Week'])
# df_tmp

# + code_folding=[0, 6, 18]
col_map_hosp = {
    'event_after_1st_dose': 'hosp_after_1st_dose',
    'event_after_2nd_dose': 'hosp_after_2nd_dose',
    'event_after_3rd_dose': 'hosp_after_3rd_dose',
    'event_for_not_vaccinated': 'hosp_unvaccinated',
}
col_map_death = {
    'event_after_1st_dose': 'death_after_1st_dose',
    'event_after_2nd_dose': 'death_after_2nd_dose',
    'event_after_3rd_dose': 'death_after_3rd_dose',
    'event_for_not_vaccinated': 'death_unvaccinated',
}
df = df_tmp[df_tmp['Type_of_event'] == 'Hospitalization'].rename(columns=col_map_hosp).drop(columns='Type_of_event')

df = df.merge(
    df_tmp[df_tmp['Type_of_event'] == 'Death'].rename(columns=col_map_death).drop(columns='Type_of_event'),
    on=['Age_group', 'Sunday_date']
)
cols = [
    'Sunday_date',
    'Age_group',
    'hosp_after_1st_dose',
    'hosp_after_2nd_dose',
    'hosp_after_3rd_dose',
    'hosp_unvaccinated',
    'death_after_1st_dose',
    'death_after_2nd_dose',
    'death_after_3rd_dose',
    'death_unvaccinated',
]
df = df[cols]
# df

# + code_folding=[0]
clean_cols =[
    'hosp_after_1st_dose',
    'hosp_after_2nd_dose',
    'hosp_after_3rd_dose',
    'hosp_unvaccinated',
    'death_after_1st_dose',
    'death_after_2nd_dose',
    'death_after_3rd_dose',
    'death_unvaccinated',
]
data_cleaning_map = {pd.NA: '0', '<5': '1'}  # TODO: check if <5 has to be mapped to sth else
df[clean_cols] = df[clean_cols].replace(data_cleaning_map).astype(float).astype(int)
df['hosp_total'] = (
    df['hosp_unvaccinated'] + df['hosp_after_1st_dose'] + df['hosp_after_2nd_dose'] + df['hosp_after_3rd_dose']
)
df['death_total'] = (
    df['death_unvaccinated'] + df['death_after_1st_dose'] + df['death_after_2nd_dose'] + df['death_after_3rd_dose']
)
df.head()
# -

# ### 2.1.2 Vaccinations

# + code_folding=[2]
df_vac = pd.read_csv(VACCINATION_DATA_PATH)

clean_cols = [
    'first_dose',
    'second_dose',
    'third_dose',
]
data_cleaning_map = {'<15': '10'}  # TODO: check if <15 has to be mapped to sth. else
df_vac[clean_cols] = df_vac[clean_cols].replace(data_cleaning_map).astype(int)

df_vac['Sunday_date'] = pd.to_datetime(
    df_vac['VaccinationDate'], format='%Y-%m-%d'
).dt.to_period('W-SAT').apply(lambda r: r.start_time) # Sunday-Monday weeks

df_vac = df_vac.rename(columns={'first_dose': '1st_dose', 'second_dose': '2nd_dose', 'third_dose': '3rd_dose'})
# df_vac
# -

# merge vaccination data
df_vac_grouped = df_vac.groupby(['Sunday_date', 'age_group']).sum().reset_index()
df = df.merge(df_vac_grouped.rename(columns={'age_group': 'Age_group'}), on=['Sunday_date', 'Age_group'])
df.head()

# ### 2.1.3 Demographic Data

# +
# load UN population data

# convert to csv, uncomment to redo
# df_pop = pd.read_excel(
#     'data/israel/WPP2019_POP_F07_1_POPULATION_BY_AGE_BOTH_SEXES.xlsx',
#     skiprows=15,
#     header=1,
#     index_col=0
# )
# df_pop = df_pop[
#     (df_pop['Region, subregion, country or area *'] == 'Israel')
#     & (df_pop['Reference date (as of 1 July)'] == 2020)
# ]

# df_pop.to_csv('data/israel/WPP2019_POP_F07_1_POPULATION_BY_AGE_BOTH_SEXES.csv')

df_pop = pd.read_csv(UN_POPULATION_DATA_PATH, index_col=0)

df_pop

# + code_folding=[10]
df_pop['0-19'] = 1000 * (df_pop['0-4'] + df_pop['5-9'] + df_pop['10-14'] + df_pop['15-19']).astype(int)
df_pop['20-29'] = 1000 * (df_pop['20-24'] + df_pop['25-29']).astype(int)
df_pop['30-39'] = 1000 * (df_pop['30-34'] + df_pop['35-39']).astype(int)
df_pop['40-49'] = 1000 * (df_pop['40-44'] + df_pop['45-49']).astype(int)
df_pop['50-59'] = 1000 * (df_pop['50-54'] + df_pop['55-59']).astype(int)
df_pop['60-69'] = 1000 * (df_pop['60-64'] + df_pop['65-69']).astype(int)
df_pop['70-79'] = 1000 * (df_pop['70-74'] + df_pop['75-79']).astype(int)
df_pop['80-89'] = 1000 * (df_pop['80-84'] + df_pop['85-89']).astype(int)
df_pop['90+'] = 1000 * (df_pop['90-94'] + df_pop['95-99'] + df_pop['100+']).astype(int)

drop_cols = [
    'Variant',
    'Region, subregion, country or area *',
    'Notes',
    'Country code',
    'Type',
    'Parent code',
    'Reference date (as of 1 July)',
    '0-4',
    '5-9',
    '10-14',
    '15-19',
    '20-24',
    '25-29',
    '30-34',
    '35-39',
    '40-44',
    '45-49',
    '50-54',
    '55-59',
    '60-64',
    '65-69',
    '70-74',
    '75-79',
    '80-84',
    '85-89',
    '90-94',
    '95-99',
    '100+',
]
df_pop = df_pop.drop(columns=drop_cols).rename(
    index={1440: 'Population_size'}).T.reset_index().rename(columns={'index': 'Age_group'})
# df_pop

# +
# rescale by current population
# source: https://www.cbs.gov.il/he/mediarelease/DocLib/2020/438/11_20_438e.pdf
current_population = 9291000
df_pop['Population_size'] = df_pop['Population_size'] * (current_population / df_pop['Population_size'].sum())
df_pop['Population_share'] = df_pop['Population_size']/current_population

df_constants = pd.DataFrame({'current_population': [current_population]})
# df_pop
# -

# merge population data into main datafram
df = df.merge(df_pop, on=['Age_group'])
df.head()

# ### 2.1.4 Infection Data

# + code_folding=[]
df_inf = pd.read_csv(CASE_DATA_PATH)
df_inf['Sunday_date'] = pd.to_datetime(df_inf['Week'].str[:10], format='%Y-%m-%d')
df_inf = df_inf.drop(columns=['Week'])

clean_cols = [
    'positive_1_6_days_after_1st_dose',
    'positive_7_13_days_after_1st_dose',
    'positive_14_20_days_after_1st_dose',
    'positive_above_20_days_after_1st_dose',
    'positive_1_6_days_after_2nd_dose',
    'positive_7_13_days_after_2nd_dose',
    'positive_14_30_days_after_2nd_dose',
    'positive_31_90_days_after_2nd_dose',
    'positive_above_3_month_after_2nd_before_3rd_dose',
    'positive_1_6_days_after_3rd_dose',
    'positive_7_13_days_after_3rd_dose',
    'positive_14_30_days_after_3rd_dose',
    'positive_31_90_days_after_3rd_dose',
    'positive_above_90_days_after_3rd_dose',
    'Sum_positive_without_vaccination',
]

data_cleaning_map = {pd.NA: '0', '<5': '4'}  # TODO: check if <5 has to be mapped to sth else
df_inf[clean_cols] = df_inf[clean_cols].replace(data_cleaning_map).astype(float).astype(int)

sum_total_cols = [
    'positive_1_6_days_after_1st_dose',
    'positive_7_13_days_after_1st_dose',
    'positive_14_20_days_after_1st_dose',
    'positive_above_20_days_after_1st_dose',
    'positive_1_6_days_after_2nd_dose',
    'positive_7_13_days_after_2nd_dose',
    'positive_14_30_days_after_2nd_dose',
    'positive_31_90_days_after_2nd_dose',
    'positive_above_3_month_after_2nd_before_3rd_dose',
    'positive_1_6_days_after_3rd_dose',
    'positive_7_13_days_after_3rd_dose',
    'positive_14_30_days_after_3rd_dose',
    'positive_31_90_days_after_3rd_dose',
    'positive_above_90_days_after_3rd_dose',
    'Sum_positive_without_vaccination',
]
df_inf['positive_total'] = df_inf[sum_total_cols].sum(axis=1)

sum_unvaccinated_cols = [
    'Sum_positive_without_vaccination',
]
df_inf['positive_unvaccinated'] = df_inf[sum_unvaccinated_cols].sum(axis=1)

sum_1st_cols = [
    'positive_1_6_days_after_1st_dose',
    'positive_7_13_days_after_1st_dose',
    'positive_14_20_days_after_1st_dose',
    'positive_above_20_days_after_1st_dose',
]
df_inf['positive_after_1st_dose'] = df_inf[sum_1st_cols].sum(axis=1)

sum_2nd_cols = [
    'positive_1_6_days_after_2nd_dose',
    'positive_7_13_days_after_2nd_dose',
    'positive_14_30_days_after_2nd_dose',
    'positive_31_90_days_after_2nd_dose',
    'positive_above_3_month_after_2nd_before_3rd_dose',
]
df_inf['positive_after_2nd_dose'] = df_inf[sum_2nd_cols].sum(axis=1)

sum_3rd_cols = [
    'positive_1_6_days_after_3rd_dose',
    'positive_7_13_days_after_3rd_dose',
    'positive_14_30_days_after_3rd_dose',
    'positive_31_90_days_after_3rd_dose',
    'positive_above_90_days_after_3rd_dose',
]
df_inf['positive_after_3rd_dose'] = df_inf[sum_3rd_cols].sum(axis=1)

df_inf['Population_share_pos'] = df_inf[
    ['Sunday_date', 'Age_group', 'positive_total']
].groupby(['Sunday_date']).apply(lambda x: x['positive_total']/x['positive_total'].sum()).values

df_inf.head()

# + code_folding=[0]
merge_cols = [
    'Sunday_date',
    'Age_group',
    'positive_total',
    'positive_unvaccinated',
    'positive_after_1st_dose',
    'positive_after_2nd_dose',
    'positive_after_3rd_dose',
    'Population_share_pos',
]
df = df.merge(df_inf[merge_cols], on=['Sunday_date', 'Age_group'], how='left')
# df
# -

# ### 2.1.5 Data Preparation

# + code_folding=[2]
# Aggregated numbers over all age groups
df_tmp = df.groupby('Sunday_date').agg(
    {
        'hosp_unvaccinated': 'sum',
        'hosp_after_1st_dose': 'sum',
        'hosp_after_2nd_dose': 'sum',
        'hosp_after_3rd_dose': 'sum',
        'hosp_total': 'sum',
        'death_unvaccinated': 'sum',
        'death_after_1st_dose': 'sum',
        'death_after_2nd_dose': 'sum',
        'death_after_3rd_dose': 'sum',
        'death_total': 'sum',
        '1st_dose': 'sum',
        '2nd_dose': 'sum',
        '3rd_dose': 'sum',
        'Population_size': 'sum',
        'Population_share': 'sum',
        'Population_share_pos': 'sum',
        'positive_total': 'sum',
        'positive_unvaccinated': 'sum',
        'positive_after_1st_dose': 'sum',
        'positive_after_2nd_dose': 'sum',
        'positive_after_3rd_dose': 'sum',
    }
).reset_index()
df_tmp['Age_group'] = 'total'
df = pd.concat([df, df_tmp]).sort_values(['Sunday_date', 'Age_group']).reset_index()
# df.head()

# +
# Number of vaccine doses relative to population size
df['1st_dose_rel'] = df['1st_dose']/df['Population_size']
df['2nd_dose_rel'] = df['2nd_dose']/df['Population_size']
df['3rd_dose_rel'] = df['3rd_dose']/df['Population_size']

# df.head()

# +
# Cumulative number of vaccine doses for each week
cols = ['Sunday_date', 'Age_group', '1st_dose', '2nd_dose', '3rd_dose']
col_map = {
    '1st_dose': '1st_dose_cum',
    '2nd_dose': '2nd_dose_cum',
    '3rd_dose': '3rd_dose_cum',
}

df = df.join(df[cols].groupby(['Age_group']).transform('cumsum').rename(columns=col_map))
df['unvaccinated_cum'] = df['Population_size'] - df['1st_dose_cum']

for col in ['1st_dose', '2nd_dose', '3rd_dose', 'unvaccinated']:
    df[col + '_cum_rel'] = df[col + '_cum'] / df['Population_size']

df.head()

# +
# Share of vaccinated under infected population
df['unvaccinated_rel_pos'] = df['positive_unvaccinated'] / df['positive_total']
df['1st_dose_rel_pos'] = df['positive_after_1st_dose'] / df['positive_total']
df['2nd_dose_rel_pos'] = df['positive_after_2nd_dose'] / df['positive_total']
df['3rd_dose_rel_pos'] = df['positive_after_3rd_dose'] / df['positive_total']

# df.head()

# + code_folding=[]
# Hospitalisation and death rates
df['hosp_unvaccinated_rel'] = df['hosp_unvaccinated']/df['unvaccinated_cum']
df['hosp_after_1st_dose_rel'] = df['hosp_after_1st_dose']/(df['1st_dose_cum'] - df['2nd_dose_cum'])
df['hosp_after_2nd_dose_rel'] = df['hosp_after_2nd_dose']/(df['2nd_dose_cum'] - df['3rd_dose_cum'])
df['hosp_after_3rd_dose_rel'] = df['hosp_after_3rd_dose']/df['3rd_dose_cum']
df['hosp_total_rel'] = df['hosp_total']/df['Population_size']

df['death_unvaccinated_rel'] = df['death_unvaccinated']/df['unvaccinated_cum']
df['death_after_1st_dose_rel'] = df['death_after_1st_dose']/(df['1st_dose_cum'] - df['2nd_dose_cum'])
df['death_after_2nd_dose_rel'] = df['death_after_2nd_dose']/(df['2nd_dose_cum'] - df['3rd_dose_cum'])
df['death_after_3rd_dose_rel'] = df['death_after_3rd_dose']/df['3rd_dose_cum']
df['death_total_rel'] = df['death_total']/df['Population_size']

# detected infection rate
df['positive_unvaccinated_rel'] = df['positive_unvaccinated']/df['unvaccinated_cum']
df['positive_after_1st_dose_rel'] = df['positive_after_1st_dose']/(df['1st_dose_cum'] - df['2nd_dose_cum'])
df['positive_after_2nd_dose_rel'] = df['positive_after_2nd_dose']/(df['2nd_dose_cum'] - df['3rd_dose_cum'])
df['positive_after_3rd_dose_rel'] = df['positive_after_3rd_dose']/df['3rd_dose_cum']
df['positive_total_rel'] = df['positive_total']/df['Population_size']

# TODO: check why there are rates > 1 (maybe related to <5 mapping)
# df['hosp_unvaccinated_rel_pos'] = (df['hosp_unvaccinated']/df['positive_unvaccinated'].shift(0)).clip(upper=1)
# df['hosp_after_1st_dose_rel_pos'] = (df['hosp_after_1st_dose']/df['positive_after_1st_dose'].shift(0)).clip(upper=1)
# df['hosp_after_2nd_dose_rel_pos'] = (df['hosp_after_2nd_dose']/df['positive_after_2nd_dose'].shift(0)).clip(upper=1)
# df['hosp_after_3rd_dose_rel_pos'] = (df['hosp_after_3rd_dose']/df['positive_after_3rd_dose'].shift(0)).clip(upper=1)
# df['hosp_total_rel_pos'] = (df['hosp_total']/df['positive_total'].shift(0)).clip(upper=1)

df['hosp_unvaccinated_rel_pos'] = (df['hosp_unvaccinated']/df['positive_unvaccinated'].shift(0))
df['hosp_after_1st_dose_rel_pos'] = (df['hosp_after_1st_dose']/df['positive_after_1st_dose'].shift(0))
df['hosp_after_2nd_dose_rel_pos'] = (df['hosp_after_2nd_dose']/df['positive_after_2nd_dose'].shift(0))
df['hosp_after_3rd_dose_rel_pos'] = (df['hosp_after_3rd_dose']/df['positive_after_3rd_dose'].shift(0))
df['hosp_total_rel_pos'] = (df['hosp_total']/df['positive_total'].shift(0))

# df['death_unvaccinated_rel_pos'] = (df['death_unvaccinated']/df['positive_unvaccinated']).clip(upper=1)
# df['death_after_1st_dose_rel_pos'] = (df['death_after_1st_dose']/df['positive_after_1st_dose']).clip(upper=1)
# df['death_after_2nd_dose_rel_pos'] = (df['death_after_2nd_dose']/df['positive_after_2nd_dose']).clip(upper=1)
# df['death_after_3rd_dose_rel_pos'] = (df['death_after_3rd_dose']/df['positive_after_3rd_dose']).clip(upper=1)
# df['death_total_rel_pos'] = (df['death_total']/df['positive_total']).clip(upper=1)

df['death_unvaccinated_rel_pos'] = (df['death_unvaccinated']/df['positive_unvaccinated'])
df['death_after_1st_dose_rel_pos'] = (df['death_after_1st_dose']/df['positive_after_1st_dose'])
df['death_after_2nd_dose_rel_pos'] = (df['death_after_2nd_dose']/df['positive_after_2nd_dose'])
df['death_after_3rd_dose_rel_pos'] = (df['death_after_3rd_dose']/df['positive_after_3rd_dose'])
df['death_total_rel_pos'] = (df['death_total']/df['positive_total'])

df.head()
# -

# ### Delta and Alpha Variant Periods

# +
df_constants['delta_0_date_from'] = datetime.datetime(year=2021, month=1, day=17)
df_constants['delta_0_date_to'] = datetime.datetime(year=2021, month=3, day=14)

df_constants['delta_1_date_from'] = datetime.datetime(year=2021, month=8, day=1)
df_constants['delta_1_date_to'] = datetime.datetime(year=2021, month=9, day=26)

df_constants
# -

# ### Save to Disc

# save to disc
df.to_pickle(OUTPUT_DIR / 'israel_df.pkl')
df_constants.to_pickle(OUTPUT_DIR / 'israel_constants.pkl')