#!/usr/bin/env python3
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import math

def mw_expert(dataset_path, T):
    data = pd.read_csv(dataset_path,header=None)
    n_servers = data.shape[0] # Number of servers
    weights = np.ones(n_servers)
    regrets = []
    eta=np.sqrt(np.log(n_servers)/T)
    for t in range(T):
        # pick a server with probability proportional to the weights
        choice = np.random.choice(n_servers, p=weights/np.sum(weights))
        # observe the loss of all servers
        losses = data.iloc[:,t].values
        # calculate the regret
        regret = losses[choice] -np.min(losses)
        regrets.append(regret)
        # update the weights of all servers
        weights = weights * np.exp(-eta* losses)
    return np.cumsum(regrets)

def mw_adversarial(dataset_path, T):
    data = pd.read_csv(dataset_path, header=None)
    n_servers = data.shape[0] # Number of servers
    weights = np.ones(n_servers)
    regrets = []
    eta = np.sqrt(np.log(n_servers) / T)
    for t in range(T):
        # calculate the probability of selecting each action i
        pi = weights / np.sum(weights)
        # select an action i with probability proportional to pi
        choice = np.random.choice(n_servers, p=pi)
        # observe the loss of the chosen action
        loss = data.iloc[choice,t]
        # calculate the regret
        regret = np.min(data.iloc[:,t].values) - loss
        regrets.append(np.abs(regret))
        # update the weights of all actions using the MW update rule
        for i in range(n_servers):
            if i == choice:
                weights[i] *= np.exp(-eta * loss / pi[i])
            else:
                weights[i] *= np.exp(eta * loss / ((1 - pi[choice]) * (n_servers - 1)))

        # renormalize the weights to ensure they sum to 1
        weights /= np.sum(weights)
    return np.cumsum(regrets)

def ucb_losses(dataset_path, T):
    data = pd.read_csv(dataset_path, header=None)
    n_servers = data.shape[0] # Number of servers
    n = np.zeros(n_servers) # Number of times each server has been selected
    losses = np.zeros(n_servers) # Cumulative losses for each server
    est_means = np.zeros(n_servers) # Estimated mean loss for each server
    regrets = np.zeros(T) # Initialize regrets array

    # Initialize the estimates and UCB values randomly
    for i in range(n_servers):
        loss = data.iloc[:,i].sum()
        n[i] += 1
        losses[i] += loss
        est_means[i] = losses[i] / n[i]

    # Run the UCB algorithm for T time steps
    for t in range(n_servers, T):
        # Choose the server with the highest UCB value
        ucb_values = est_means + np.sqrt(2 * np.log(t) / (n + 1e-6))
        server = np.argmax(ucb_values)

        # Update the estimates and UCB values
        loss = data.iloc[server,t]
        n[server] += 1
        losses[server] -= loss
        est_means[server] = losses[server] / n[server]

        # Calculate and record the regret
        optimal_loss = data.iloc[:,t].min()
        regret = optimal_loss - loss
        regrets[t] = np.abs(regret)

    # Calculate and return the cumulative regrets
    return np.cumsum(regrets)

# Compare performance of MW algorithm in expert and bandit cases
T = [1000, 7000]
for t in T:
    regrets_expert = mw_expert("Milano_timeseries.csv",t)
    regrets_bandit = mw_adversarial("Milano_timeseries.csv",t)
    regrets_ucb = ucb_losses("Milano_timeseries.csv",t)

    plt.figure(figsize=(8, 6))
    plt.plot(regrets_expert, label='MW Expert')
    plt.plot(regrets_bandit, label='MW Bandit')
    plt.plot(regrets_ucb, label='UCB')
    # plt.semilogy(regrets_expert, label='MW Expert')
    # plt.semilogy(regrets_bandit, label='MW Bandit')
    # plt.semilogy(regrets_ucb, label='UCB')
    plt.legend()
    plt.title(f"T={t}")
    plt.xlabel('Time')
    plt.ylabel('Cumulative regret')
    plt.show()


import re
import csv
import math
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest

def score(df, x):
    col = df[[str(x)]]
    col = col.drop(col[np.isnan(col[str(x)])].index)
    col = np.array(col)
    col.reshape(-1, 1)
    c = int(col.shape[0])
    # 如果整个属性无人评分
    if (c == 0):
        return (np.nan)
    model = IsolationForest(n_estimators=100,
                            max_samples='auto',
                            contamination=float(0.2),
                            max_features=1.0)
    model.fit(col)
    # depth：可信度（-不可信度），代表可信程度
    depth = -model.decision_function(col)

    # 归一化
    M = depth.max()
    m = depth.min()
    d_normal = np.zeros(depth.shape)
    for i in range(depth.shape[0]):
        d_normal[i] = (depth[i] - m) / (M - m)
    # 求得分
    i = 0
    j = 0
    h = 0
    while i < c:
        h = h + float(d_normal[i])
        j = j + float(col[i]) * float(d_normal[i])
        i = i + 1
    # df1=df[str(x)].describe()
    return (j / h)

x = 1
lista = []
df = pd.read_csv('./normaldata.csv')
while x <= 26:
    lista.append([])
    lista[x - 1].append('attribute%s' % str(x))
    lista[x - 1].append(score(df, x))
    out = open(r'./output.csv', 'a', newline='')
    csv_write = csv.writer(out, dialect='excel')
    csv_write.writerow(lista[x - 1])
    x = x + 1
out.close()
print(lista)


import cv2
import time
import os

SAMPLE_VIDEO_PATH = os.environ["SAMPLE_VIDEO_PATH"]

frames  = []

start = time.time()

video_source = cv2.VideoCapture(SAMPLE_VIDEO_PATH)
while video_source.isOpened():
    ret, frame = video_source.read()
    if not ret:
        break
    frames.append(frame)

end = time.time()
print(f"Total time taken for reading the complete video is : {abs(end-start)} seconds")

for frame in frames:
    cv2.imshow("Frame", frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

video_source.release()
cv2.destroyAllWindows()


from django import template
from django.utils.safestring import mark_safe

register = template.Library()

def _review_score_number(context, score):
    if score is None:
        return '×'

    max_score = context['request'].event.settings.get('review_max_score') if context else None
    if isinstance(score, int) or (isinstance(score, float) and score.is_integer()):
        score = int(score)
        name = context and context['request'].event.settings.get(f'review_score_name_{score}')
        if name:
            return f'{score}/{max_score} (“{name}”)'
    else:
        score = round(score, 1)

    return f'{score}/{max_score}'

def _review_score_override(positive_overrides, negative_overrides):
    result = ''
    if positive_overrides:
        result += f''
        if positive_overrides > 1 or negative_overrides:
            result += f' {positive_overrides}'
    if negative_overrides:
        result += f''
        if negative_overrides > 1 or positive_overrides:
            result += f' {negative_overrides}'
    return result

@register.simple_tag(takes_context=True)
def review_score(context, submission):
    score = submission.current_score
    if score is None:
        return '-'
    if hasattr(submission, 'has_override') and not submission.has_override:
        return _review_score_number(context, score)
    positive_overrides = submission.reviews.filter(override_vote=True).count()
    negative_overrides = submission.reviews.filter(override_vote=False).count()
    if positive_overrides or negative_overrides:
        return mark_safe(_review_score_override(positive_overrides, negative_overrides))
    return _review_score_number(context, score)


import stat

import pytest
from ruamel import yaml

from ansiblediscover.utils.fs import FS

def test_load_yaml_raises_on_invalid_yaml(tmpdir):
    content = 'this: is: invalid'

    task_file = tmpdir.join("mydata.yml")
    task_file.write(content)

    with pytest.raises(TypeError):
        FS.load_yaml(str(task_file))

def test_load_yaml_raises_on_missing_file(tmpdir):
    task_file = tmpdir.join('mydata.yml')

    with pytest.raises(OSError):
        FS.load_yaml(str(task_file))

def test_load_yaml_raises_on_wrong_file_permissions(tmpdir):
    task_file = tmpdir.join("mydata.yml")
    task_file.write('irrelevant')
    task_file.chmod(stat.S_IWUSR)

    with pytest.raises(OSError):
        FS.load_yaml(str(task_file))

def test_load_yaml_succeeds_on_valid_yaml(tmpdir):
    content = ['hello', {'this': 'is valid yaml'}]

    task_file = tmpdir.join("mydata.yml")
    task_file.write(yaml.dump(content))

    assert content == FS.load_yaml(str(task_file))

def test_load_yaml_succeeds_on_tags(tmpdir):
    def default_ctor(loader, tag_suffix, node):
        return tag_suffix + ' ' + node.value

    yaml.add_multi_constructor('', default_ctor)
    content = yaml.load("""\
        data_contains: !vault |
            $ANSIBLE_VAULT;1.1;AES256
            1234567890ABCDEFGHIJKLMNO
    """)

    task_file = tmpdir.join("mydata.yml")
    task_file.write(yaml.dump(content))

    assert content == FS.load_yaml(str(task_file))


from django.urls import path
from .views import *

urlpatterns = [
    path('techservice/', TechserviceListView.as_view(), name='techservice_list'),
    path('claim/', ClaimListView.as_view(), name='claim_list'),
    path('vehicle/<pk>/techservice', TechserviceDetailView.as_view(), name='techservice_detail'),
    path('vehicle/<pk>/claim', ClaimDetailView.as_view(), name='claim_detail'),
    path('techservice/create', TechserviceCreateView.as_view(), name='techservice_create'),
    path('claim/create', ClaimCreateView.as_view(), name='claim_create'),
    path('techservice/<pk>/update', TechserviceUpdateView.as_view(), name='techservice_update'),
    path('claim/<pk>/update', ClaimUpdateView.as_view(), name='claim_update'),
    path('techservice/<pk>/delete', TechserviceDeleteView.as_view(), name='techservice_delete'),
    path('claim/<pk>/delete', ClaimDeleteView.as_view(), name='claim_delete'),
    path('techservice/<pk>/description/<atribute>', TechserviceDetailsView.as_view(),
         name='techservice_details'),
    path('claim/<pk>/description/<atribute>', ClaimDetailsView.as_view(), name='claim_details'),

    path('', IndexView.as_view(template_name='/index.html'), name='index'),
    path('search/', VehicleSearchView.as_view(), name='vehicle_search'),
    path('vehicles/', VehicleListView.as_view(), name='vehicle_list'),
    path('vehicle/<pk>/detail', VehicleDetailView.as_view(), name='vehicle_detail'),
    path('vehicle/create', VehicleCreateView.as_view(), name='vehicle_create'),
    path('vehicle/<pk>/update', VehicleUpdateView.as_view(), name='vehicle_update'),
    path('vehicle/<pk>/delete', VehicleDeleteView.as_view(), name='vehicle_delete'),
    path('vehicle/<pk>/description/<atribute>', VehicleDetailsView.as_view(), name='vehicle_details'),
]


# https://github.com/Practical-CV/EYE-BLINK-DETECTION-WITH-OPENCV-AND-DLIB

from scipy.spatial import distance as dist
from imutils import face_utils

import numpy as np

import time
import dlib
import os
import cv2

# load facial landmark detector
model_root = os.getcwd() + '/models'

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(model_root + "/shape_predictor_68_face_landmarks_GTX.dat")

(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS['left_eye']
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS['right_eye']

# cnn detector + predictor
import caffe
os.environ['GLOG_minloglevel'] = '2'

model_weight = model_root + "/mmod_human_face_detector.dat"
detector_cnn = dlib.cnn_face_detection_model_v1(model_weight)

net_work_path = model_root + '/landmark_deploy.prototxt'
weight_path = model_root + '/VanFace.caffemodel'

net = caffe.Net(net_work_path, weight_path, caffe.TEST)
net.name = 'FaceThink_face_landmark'

def current_time():
    return int(round(time.time() * 1000))

def eye_aspect_ratio(eye):
    A = dist.euclidean(eye[1], eye[5])
    B = dist.euclidean(eye[2], eye[4])
    C = dist.euclidean(eye[0], eye[3])

    return (A + B) / (2.0 * C)

def face_rect(frame, rect):

    x1 = max(0, rect.left())
    y1 = max(0, rect.top())
    x2 = min(rect.right(), frame.shape[1])
    y2 = min(rect.bottom(), frame.shape[0])

    gray = frame[y1:y2+1, x1:x2+1,]
    # gray = cv2.cvtColor(roi, cv2.COLOR_GRAY2GRAY)

    w, h = 60, 60
    res = cv2.resize(gray, (w, h), 0.0, 0.0, interpolation=cv2.INTER_CUBIC)
    resize_mat = np.float32(res)

    m = np.zeros((w, h))
    sd = np.zeros((w, h))
    mean, std_dev = cv2.meanStdDev(resize_mat, m, sd)
    new_m = mean[0][0]
    new_sd = std_dev[0][0]
    img = (resize_mat - new_m) / (0.000001 + new_sd)

    return img

# get 68 points
def landmarks(points, rect, frame):
    x1 = max(0, rect.left())
    y1 = max(0, rect.top())
    x2 = min(rect.right(), frame.shape[1])
    y2 = min(rect.bottom(), frame.shape[0])

    res = []
    for i in range(len(points) // 2):
        x = points[2*i] * (x2 - x1) + x1
        y = points[2*i+1] * (y2 - y1) + y1
        res.append([x, y])

    return np.array(res)

def extract_ears(img, grayed=False, cnn=False, debug=False):
    gray = img
    if not grayed:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    start_ms = current_time()
    ear = -1.0
    center = np.array([.0, .0])

    if cnn:
        rects = [r.rect for r in detector_cnn(gray, 0)]
    else:
        rects = detector(gray, 0)

    # take only the first
    for rect in rects[:1]:
        # if cnn:
        #     net.blobs['data'].data[...] = face_rect(gray, rect)
        #     net.forward()
        #     shape = net.blobs['Dense3'].data[0].flatten()
        #     shape = landmarks(shape, rect, gray)

        shape = predictor(gray, rect)
        shape = face_utils.shape_to_np(shape)

        leftEye = shape[lStart:lEnd]
        rightEye = shape[rStart:rEnd]

        leftEAR = eye_aspect_ratio(leftEye)
        rightEAR = eye_aspect_ratio(rightEye)

        l_center = sum(leftEye) / len(leftEye)
        r_center = sum(rightEye) / len(rightEye)

        ear = (leftEAR + rightEAR) / 2.0
        center = (l_center + r_center) / 2.0

        if debug:
            leftEyeHull = cv2.convexHull(leftEye)
            rightEyeHull = cv2.convexHull(rightEye)

            cv2.drawContours(img, [leftEyeHull], -1, (0, 255, 0), 1)
            cv2.drawContours(img, [rightEyeHull], -1, (0, 255, 0), 1)

        duration_ms = current_time() - start_ms
        print("Eye aspect ratio took:     ", str(duration_ms / 1000) + "s")

    return ear, center


from datetime import datetime, timedelta
import os
import operator
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.python_operator import PythonOperator
from operators.stage_from_s3 import StageFromS3Operator
from operators.get_data import GetDataOperator
from operators.load_fact import LoadFactOperator
from operators.data_quality import DataValidationOperator
from airflow.hooks.S3_hook import S3Hook
from helpers import LoadFactQueries, LoadDimensionQueries

# Arguments for this dag
default_args = {
    'owner': 'hiroo',
    'start_date': datetime(2021,1,3),
    'depends_on_past': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=3),
    'catchup': False,
    'email':['kifa0422@gmail.com'],
    'email_on_failure':False,
    'email_on_retry':False
}

# dag definition, the dag is scheduled to run daily
dag = DAG('covid_warehouse_dag',
          default_args=default_args,
          description='Load and transform data in Redshift with Airflow',
          schedule_interval= '@daily'
        )

# address for temporary storage in disk
OUTPUT_PATH = '/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables'

# helper function to move files to AWS S3
def upload_to_S3(filename, key, bucket_name):
	hook = S3Hook('my_S3_conn')
	hook.load_file(filename, key, bucket_name, replace=True)

start_operator = DummyOperator(task_id='Begin_execution',  dag=dag)

# get data about county's cases, make copy in local disk
get_county_case = GetDataOperator(
		task_id='Get_county_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_county_case',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/latimes-county-totals.csv',
		output_path= OUTPUT_PATH
	)

# get data about hospitalized cases
get_hospitalized_case = GetDataOperator(
		task_id='Get_hospitalized_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_hospitalized',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-hospital-patient-county-totals.csv',
		output_path=OUTPUT_PATH
	)

# get data about every nursing home cases
get_nursing_home = GetDataOperator(
		task_id='Get_nursing_home_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_nursing_home',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-skilled-nursing-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about senior home cases
get_senior_home = GetDataOperator(
		task_id='Get_senior_home_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_senior_facilities',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-adult-and-senior-care-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about specific city/places
get_place_data = GetDataOperator(
		task_id='Get_place_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_place_totals',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-adult-and-senior-care-facilities.csv',
		output_path=OUTPUT_PATH
	)

# get data about every prison's cases
get_prison_case_data = GetDataOperator(
		task_id='Get_prison_case_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_prison_cases',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdcr-prison-totals.csv',
		output_path=OUTPUT_PATH
	)

# get data about reopening metrics
get_reopening_metrics = GetDataOperator(
		task_id='Get_reopening_metrics_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_reopening_metrics',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-reopening-metrics.csv',
		output_path=OUTPUT_PATH
	)

# get data about nationwide cases
get_nationwide_case = GetDataOperator(
		task_id='Get_nationwide_data',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='nationwide_cases',
		data_url='https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv',
		output_path=OUTPUT_PATH
	)

# get data about prioson's information, used for dimension tables
get_prison_info = GetDataOperator(
		task_id='Get_prison_info',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_prisons',
		data_url='https://raw.githubusercontent.com/nrjones8/cdcr-population-data/master/data/monthly_cdcr_population.csv',
		output_path=OUTPUT_PATH
	)

# get data about reopening tier for each county
get_reopening_tier = GetDataOperator(
		task_id='Get_reopening_tier',
		dag=dag,
		postgres_conn_id='local_postgres',
		table='staging_reopening_tier',
		data_url='https://raw.githubusercontent.com/datadesk/california-coronavirus-data/master/cdph-reopening-tiers.csv',
		output_path=OUTPUT_PATH
	)

# move county data to S3
county_case_toS3 = PythonOperator(
		task_id='Upload_county_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':'/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables/staging_county_case.csv',
			'key':'tables/staging_county_case.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move hospitalized case data to S3
hospitalized_case_toS3 = PythonOperator(
		task_id='Upload_hospitalized_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':'/Users/hiroakioshima/Desktop/Hiro_Covid_Project/tables/staging_hospitalized.csv',
			'key':'tables/staging_hospitalized.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move nursing home data to S3
nursing_case_toS3 = PythonOperator(
		task_id='Upload_nursinghome_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_nursing_home.csv',
			'key':'tables/staging_nursing_home.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move senior cases to S3
senior_case_toS3 = PythonOperator(
		task_id='Upload_senior_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_senior_facilities.csv',
			'key':'tables/staging_senior_facilities.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move place data to S3
place_totals_tos3 = PythonOperator(
		task_id='Upload_places_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_place_totals.csv',
			'key':'tables/staging_place_totals.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move prison case data to S3
prison_case_tos3 = PythonOperator(
		task_id='Upload_prison_case_s3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_prison_cases.csv',
			'key':'tables/staging_prison_cases.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move reopening data to S3
reopening_metrics_tos3 = PythonOperator(
		task_id='Upload_reopening_metrics_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_reopening_metrics.csv',
			'key':'tables/staging_reopening_metrics.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move nationwide case data to S3
nationwide_tos3 = PythonOperator(
		task_id='Upload_nationwide_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/nationwide_cases.csv',
			'key':'tables/nationwide_cases.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move prison informaiton data to S3
prison_info_tos3 = PythonOperator(
		task_id='Upload_prisoninfo_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_prisons.csv',
			'key':'tables/staging_prisons.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# move reopening tier data to S3
reopening_tier_tos3 = PythonOperator(
		task_id='Upload_reopening_tier_tos3',
		dag=dag,
		python_callable=upload_to_S3,
		op_kwargs={
			'filename':OUTPUT_PATH + '/staging_reopening_tier.csv',
			'key':'tables/staging_reopening_tier.csv',
			'bucket_name':'hiro-covid-datalake'
		}
	)

# load hospitalized data to staging table on redshift server
load_hospitalized_redshift = StageFromS3Operator(
		task_id='Load_hospitalized_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_hospitalized',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_hospitalized.csv'
	)

# load county case data to staging table on redshift server
load_county_redshift = StageFromS3Operator(
		task_id='Load_county_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_county_case',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_county_case.csv'
	)

# load nursing home cases data to staging table on redshift server
load_nursing_redshift = StageFromS3Operator(
		task_id='Load_nursing_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_nursing_home',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_nursing_home.csv'
	)

# load senior home cases data to staging table on redshift server
load_seniorhome_redshift = StageFromS3Operator(
		task_id='Load_senior_to_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_senior_facilities',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_senior_facilities.csv'
	)

# load place data to staging table on redshift server
load_places_redshift = StageFromS3Operator(
		task_id='Load_places_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_place_totals',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_place_totals.csv'
	)

# load prison cases data to staging table on redshift server
load_prisoncase_redshift = StageFromS3Operator(
		task_id='Load_prisoncase_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_prison_cases',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_prison_cases.csv'
	)

# load openmetrics data to staging table on redshift server
load_openmetrics_redshift = StageFromS3Operator(
		task_id='Load_openmetrics_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_reopening_metrics',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_reopening_metrics.csv'
	)

# load nationwide cases data to staging table on redshift server
load_nationwide_redshift = StageFromS3Operator(
		task_id='Load_nationwide_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='nationwide_cases',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/nationwide_cases.csv'
	)

# load prison information data to staging table on redshift server
load_prisoninfo_redshift = StageFromS3Operator(
		task_id='Load_prisoninfo_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_prisons',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_prisons.csv'
	)

# load open tier data to staging table on redshift server
load_opentier_redshift = StageFromS3Operator(
		task_id='Load_opentier_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_reopening_tier',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='tables/staging_reopening_tier.csv'
	)

# load information about healthcare faculty from data lake to redshift staging table
load_healthcarefacs_redshift = StageFromS3Operator(
		task_id='Load_healthcarefacs_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='staging_healthcare_listing',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='static/healthcare_facility_locations.csv'
	)

# load census data from data lake to redshift staging table
load_census_redshift = StageFromS3Operator(
		task_id='Load_census_redshift',
	    dag=dag,
	    redshift_conn_id='redshift',
	    table='census_data2019',
	    aws_credentials_id='aws_credentials',
	    S3_BUCKET='hiro-covid-datalake',
	    s3_key='static/census_2019.csv'
	)

# transform the staging tables and load county case fact table
loadfact_county = LoadFactOperator(
		task_id='Loadfact_county',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'county_case',
		source_table = 'staging_county_case',
		sql = LoadFactQueries.load_county_case
	)

# transform the staging tables and load the hospitalized fact table on redshift
loadfact_hospitalized = LoadFactOperator(
		task_id='Loadfact_hospitalized',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'hospitalized_case',
		source_table = 'staging_hospitalized',
		sql = LoadFactQueries.load_hospitalized_case
	)

# transform the staging tables and load nursing home cases fact table on redshift
loadfact_nursing = LoadFactOperator(
		task_id='Loadfact_nursing',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'nursing_home_case',
		source_table = 'staging_nursing_home',
		sql = LoadFactQueries.load_nursing_home
	)

# transform the staging tables and load senior home cases fact table on redshift
loadfact_senior = LoadFactOperator(
		task_id='Loadfact_senior',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'senior_facility_case',
		source_table = 'staging_senior_facilities',
		sql = LoadFactQueries.load_senior_facs
	)

# transform the staging tables and load place cases fact table on redshift
loadfact_place = LoadFactOperator(
		task_id='Loadfact_place',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'place_total',
		source_table = 'staging_place_totals',
		sql = LoadFactQueries.load_places
	)

# transform the staging tables and load prison cases fact table on redshift
loadfact_prison = LoadFactOperator(
		task_id='Loadfact_prison',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'prison_case',
		source_table = 'staging_prison_cases',
		sql = LoadFactQueries.load_prison
	)

# transform the staging tables and load reopening metrics fact table on redshift
loadfact_open_metrics = LoadFactOperator(
		task_id='Loadfact_open_metrics',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'reopening_metrics',
		source_table = 'staging_reopening_metrics',
		sql = LoadFactQueries.load_open_metrics
	)

# transform the staging tables and load reopening tiers fact table on redshift
loadfact_open_tiers = LoadFactOperator(
		task_id='Loadfact_open_tiers',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'reopening_tier',
		source_table = 'staging_reopening_tier',
		sql = LoadFactQueries.load_open_tiers
	)

# transform the staging tables and load nationwide cases fact table on redshift
loadfact_nationwide = LoadFactOperator(
		task_id='Loadfact_nationwide',
		dag=dag,
		redshift_conn_id='redshift',
		table = 'other_states_cases',
		source_table = 'nationwide_cases',
		sql = LoadFactQueries.load_nationwide
	)

# transform the staging tables and load healthcare facilities dimension table on redshift
loaddim_healthcare = PostgresOperator(
	task_id='Loaddim_healthcare',
    dag=dag,
    sql=LoadDimensionQueries.load_healthcare_facs,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load county dimension table on redshift
loaddim_county = PostgresOperator(
	task_id='Loaddim_county',
    dag=dag,
    sql=LoadDimensionQueries.load_county,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load prison dimension table on redshift
loaddim_prison = PostgresOperator(
	task_id='Loaddim_prison',
    dag=dag,
    sql=LoadDimensionQueries.load_prison,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load nursing homes dimension table on redshift
loaddim_nursing = PostgresOperator(
	task_id='Loaddim_nursing',
    dag=dag,
    sql=LoadDimensionQueries.load_nursing,
    postgres_conn_id='redshift'
	)

# transform the staging tables and load dimension table on redshift
loaddim_senior = PostgresOperator(
	task_id='Loaddim_senior',
    dag=dag,
    sql=LoadDimensionQueries.load_senior,
    postgres_conn_id='redshift'
	)

# perfom data validation on fact tables, check if the latest data is loaded correctly
dataval_fact = DataValidationOperator(
	task_id='data_validation_facttables',
	dag=dag,
	redshift_conn_id='redshift',
	dq_checks= [{'testsql':'SELECT MAX(date) FROM county_case', 'expected':datetime(2021,1,20), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM hospitalized_case', 'expected':datetime(2021,1,20), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM reopening_metrics', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM nursing_home_case', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM senior_facility_case', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM place_total', 'expected':datetime(2021,1,21), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM prison_case', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM reopening_tier', 'expected':datetime(2021,1,22), 'op':operator.eq},
			   {'testsql':'SELECT MAX(date) FROM other_states_cases', 'expected':datetime(2021,1,21), 'op':operator.eq}]
	)

# perform data validation on dimension tables, check if the number of records matches the expected number for each table
dataval_dim = DataValidationOperator(
	task_id='data_validation_dimtables',
	dag=dag,
	redshift_conn_id='redshift',
	dq_checks= [{'testsql':'SELECT COUNT(*) FROM county', 'expected':50, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM healthcare_facility', 'expected':1000, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM prison', 'expected':200, 'op':operator.le},
			   {'testsql':'SELECT COUNT(*) FROM nursing_facility', 'expected':200, 'op':operator.ge},
			   {'testsql':'SELECT COUNT(*) FROM adult_senior_care', 'expected':200, 'op':operator.ge}]
	)

end_operator = DummyOperator(task_id='Stop_execution',  dag=dag)

# Dependency of this dag
start_operator >> [get_county_case, get_hospitalized_case, get_nursing_home, get_senior_home, get_place_data, \
					get_prison_case_data, get_reopening_metrics, get_nationwide_case, get_prison_info, get_reopening_tier]

start_operator >> [load_healthcarefacs_redshift, load_census_redshift]

get_county_case >> county_case_toS3 >> load_county_redshift >> loadfact_county
get_hospitalized_case >> hospitalized_case_toS3 >> load_hospitalized_redshift >> loadfact_hospitalized
get_nursing_home >> nursing_case_toS3 >> load_nursing_redshift >> loadfact_nursing
get_senior_home >> senior_case_toS3 >> load_seniorhome_redshift >> loadfact_senior
get_place_data >> place_totals_tos3 >> load_places_redshift >> loadfact_place
get_prison_case_data >> prison_case_tos3 >> load_prisoncase_redshift >> loadfact_prison
get_reopening_metrics >> reopening_metrics_tos3 >> load_openmetrics_redshift >> loadfact_open_metrics
get_nationwide_case >> nationwide_tos3 >> load_nationwide_redshift >> loadfact_nationwide
get_prison_info >> prison_info_tos3 >> load_prisoninfo_redshift
get_reopening_tier >> reopening_tier_tos3 >> load_opentier_redshift >> loadfact_open_tiers

[loadfact_county, loadfact_hospitalized, loadfact_nursing, loadfact_senior, loadfact_place, loadfact_prison, \
 loadfact_open_metrics, loadfact_nationwide, loadfact_open_tiers] >> dataval_fact

[loaddim_healthcare, loaddim_county, loaddim_prison, loaddim_nursing, loaddim_senior] >> dataval_dim

[dataval_fact, dataval_dim] >> end_operator

[load_census_redshift, loadfact_open_tiers, load_healthcarefacs_redshift] >> loaddim_county
load_healthcarefacs_redshift >> [loaddim_healthcare, loaddim_nursing, loaddim_senior]
[load_prisoninfo_redshift, load_prisoncase_redshift] >> loaddim_prison


#!/usr/bin/env python2.7
import tweepy
from PIL import Image
import numpy
import random

# Open the file containing all the things.
fid = open('colors.csv', 'r')
names = fid.readlines()

# Pick one at random
thing = random.sample(names, 1)[0]
colourName = thing.split(",")[1]
colourRGB = thing.split(",")[2]
fileName = "temp.png"

colourString = "0xFF"
colourString += format(int(thing.split(",")[5]),'X').zfill(2) + format(int(thing.split(",")[4]),'X').zfill(2) + format(int(thing.split(",")[3]),'X').zfill(2)

w,h=1024,768
img = numpy.empty((w,h),numpy.uint32)
img.shape=h,w

img[0:768,0:1024]=format(int(colourString,16))

pilImage = Image.frombuffer('RGBA',(w,h),img,'raw','RGBA',0,1)
pilImage.save(fileName)

#print "SAVED!"
# Send the tweet with photo
# Consumer keys and access tokens, used for OAuth
consumer_key        = ''
consumer_secret     = ''
access_token        = ''
access_token_secret = ''

# OAuth process, using the keys and tokens
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

# Creation of the actual interface, using authentication
api = tweepy.API(auth)
status = colourName + "\n" + colourRGB + "\n"
api.update_with_media(fileName, status=status)


import json
from urllib.request import proxy_bypass

from constants.constants import FORBIDDEN_REQUEST_FOUND, MODEL_ALREADY_EXIST, MODEL_CREATION_FAILED, MODEL_PARAM_MISSED, MODEL_RECORD_NOT_FOUND
from core.models import  Batch, Group, ProjectTitle, TopProject,User, Voter
from django.db import transaction
from django_filters import rest_framework as filters
from django_filters.rest_framework import DjangoFilterBackend
from pkg.util import error_response, success_response
from rest_framework.decorators import action
from rest_framework import viewsets
from rest_framework.filters import OrderingFilter, SearchFilter
from rest_framework.response import Response
from core.permissions import IsAdmin
from django.db.models import F
from rest_framework.decorators import action
from django.db.models import  Sum
# from django.core import serializers

from .serializer import TopProjectSerializer,VoterSerializer
class TopProjectFilter(filters.FilterSet):
    project_id = filters.NumberFilter(field_name="project_id")
    user_id = filters.NumberFilter(field_name="user_id")
    group = filters.NumberFilter(field_name="group")
    batch = filters.CharFilter(field_name="batch")

    class Meta:
        model = TopProject
        fields = ["title", "batch", "group","doc_path","vote","description","is_approved"]

class TopProjectViewSet(viewsets.ModelViewSet):
    serializer_class = TopProjectSerializer
    filter_backends = [DjangoFilterBackend, OrderingFilter, SearchFilter]
    filter_class = TopProjectFilter
    ordering_fields = ["batch", "group"]
    search_fields = ["batch", "group"]
    filterset_fields = [
        "batch","group","id"
    ]
    queryset=None
    def get_queryset(self):
        print("fetching ...")
        queryset = TopProject.objects.all().order_by('id')
        return queryset

    @transaction.atomic
    def create(self, request, *args, **kwargs):
        print("creating ...")
        form_data = request.data
        global user_obj
        global project_obj
        global batch_obj
        global title_obj
        global group_obj
        usr=self.request.user.id
        print("user id =>",usr)
        try:
            user_obj = User.objects.get(id=int(usr))
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            batch_obj = Batch.objects.get(name=form_data["batch"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Batch")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            group_obj = Group.objects.get(id=form_data["group"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Group")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        try:
            title_obj = ProjectTitle.objects.get(id=form_data["id"])
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "Title")
            return Response(res,status=int(res['status_code']), content_type="application/json")

        project_obj = TopProject.objects.create(
            group=group_obj,
            batch=batch_obj,
            title=title_obj,
            description=form_data["description"]
        )
        serializer = TopProjectSerializer(project_obj)
        data = success_response(serializer.data)
        return Response((data))

    def retrieve(self, request, pk=None):
        if not pk.isdigit():
            res = error_response(request, MODEL_PARAM_MISSED, "TopProject")
            res['message']='Invalid request parameter found.'
            return Response(res,status=int(res['status_code']), content_type="application/json")
        try:
            queryset = TopProject.objects.get(pk=pk)
        except:
            res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
            res['message']='TopProject not found with id '+pk+"."
            return Response(res,status=int(res['status_code']), content_type="application/json")
        serializer=TopProjectSerializer(queryset)
        return Response(serializer.data)
    @action(
        detail=True,
        methods=["PUT"],
        permission_classes=[IsAdmin],
        url_path="vote",
    )
    def vote(self, request,*args, **kwargs):
        print("updating ...")
        pk = int(kwargs["pk"])
        usr=self.request.user.id
        voter_count=0
        top_project_obj = None
        user_obj = None
        voter_count = Voter.objects.filter(user_id=usr).count()
        if voter_count==1:
            voter = Voter.objects.get(user_id=usr)
            # print("User Information => ","user_id ->",voter.user_id.id,"project_id ->",voter.project_id.id,"pk ->",pk," voter.project_id.id != pk ->" ,voter.project_id.id != pk)
            if voter.user_id.id == usr and voter.project_id.id == pk:
                print("line-106")
                TopProject.objects.filter(id=pk).update(vote=F('vote')-1)
                Voter.objects.filter(user_id=usr,project_id=pk).delete()
                return Response({"message":"You unvoted successfully!"})

            elif (voter.user_id.id == usr and voter.project_id.id != pk):
                print("line-111")
                TopProject.objects.filter(id=voter.project_id.id).update(vote=F('vote')-1)
                Voter.objects.filter(user_id=voter.user_id.id,project_id=voter.project_id.id).delete()
                print("line-113")
                try:
                    user_obj = User.objects.get(id=usr)
                except:
                    res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
                    return Response(res,status=int(res['status_code']), content_type="application/json")
                try:
                    top_project_obj = TopProject.objects.get(id=pk)
                except:
                    res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
                    return Response(res,status=int(res['status_code']), content_type="application/json")
                Voter.objects.create(user_id=user_obj,project_id=top_project_obj)
                TopProject.objects.filter(id=pk).update(vote=F('vote')+1)
                return Response({"message":"You Voted Successfully!"})

            else:
                print("line-132")
                pass
        else:
            print("line-119")
            top_project_obj = TopProject.objects.filter(id=pk).update(vote=F('vote')+1)
            try:
                user_obj = User.objects.get(id=usr)
            except:
                res = error_response(request, MODEL_RECORD_NOT_FOUND, "User")
                return Response(res,status=int(res['status_code']), content_type="application/json")
            try:
                top_project_obj = TopProject.objects.get(id=pk)
            except:
                res = error_response(request, MODEL_RECORD_NOT_FOUND, "TopProject")
                return Response(res, status=int(res['status_code']),content_type="application/json")
            Voter.objects.create(user_id=user_obj,project_id=top_project_obj)
            return Response({"message":"You Voted Successfully!"})

    def destroy(self, request, *args, **kwargs):
        print("deleting ...")
        instance = TopProject.objects.filter(id=int(kwargs["pk"]))
        if len(instance) != 1:
            res = error_response(self.request, MODEL_RECORD_NOT_FOUND, "TopProject")
            return Response(res, status=int(res['status_code']),content_type="application/json")
        instance.delete()
        return Response({"result": "TopProject instance was successfuly deleted!"})


import re, sys, json

from jasy.js.tokenize.Lang import keywords
from jasy.js.parse.Lang import expressions, futureReserved

all = [ "Compressor" ]

high_unicode = re.compile(r"\\u[2-9A-Fa-f][0-9A-Fa-f]{3}")
ascii_encoder = json.JSONEncoder(ensure_ascii=True)
unicode_encoder = json.JSONEncoder(ensure_ascii=False)

#
# Class
#

class Compressor:
    __semicolonSymbol = ";"
    __commaSymbol = ","

    def __init__(self, format=None):
        if format:
            if format.has("semicolon"):
                self.__semicolonSymbol = ";\n"

            if format.has("comma"):
                self.__commaSymbol = ",\n"

        self.__forcedSemicolon = False

    #
    # Main
    #

    def compress(self, node):
        type = node.type
        result = None

        if type in self.__simple:
            result = type
        elif type in self.__prefixes:
            if getattr(node, "postfix", False):
                result = self.compress(node[0]) + self.__prefixes[node.type]
            else:
                result = self.__prefixes[node.type] + self.compress(node[0])

        elif type in self.__dividers:
            first = self.compress(node[0])
            second = self.compress(node[1])
            divider = self.__dividers[node.type]

            # Fast path
            if node.type not in ("plus", "minus"):
                result = "%s%s%s" % (first, divider, second)

            # Special code for dealing with situations like x + ++y and y-- - x
            else:
                result = first
                if first.endswith(divider):
                    result += " "

                result += divider

                if second.startswith(divider):
                    result += " "

                result += second

        else:
            try:
                result = getattr(self, "type_%s" % type)(node)
            except KeyError:
                print("Compressor does not support type '%s' from line %s in file %s" % (type, node.line, node.getFileName()))
                sys.exit(1)

        if getattr(node, "parenthesized", None):
            return "(%s)" % result
        else:
            return result

    #
    # Helpers
    #

    def __statements(self, node):
        result = []
        for child in node:
            result.append(self.compress(child))

        return "".join(result)

    def __handleForcedSemicolon(self, node):
        if node.type == "semicolon" and not hasattr(node, "expression"):
            self.__forcedSemicolon = True

    def __addSemicolon(self, result):
        if not result.endswith(self.__semicolonSymbol):
            if self.__forcedSemicolon:
                self.__forcedSemicolon = False

            return result + self.__semicolonSymbol

        else:
            return result

    def __removeSemicolon(self, result):
        if self.__forcedSemicolon:
            self.__forcedSemicolon = False
            return result

        if result.endswith(self.__semicolonSymbol):
            return result[:-len(self.__semicolonSymbol)]
        else:
            return result

    #
    # Data
    #

    __simple_property = re.compile(r"^[a-zA-Z_$][a-zA-Z0-9_$]*$")
    __number_property = re.compile(r"^[0-9]+$")

    __simple = ["true", "false", "null", "this", "debugger"]

    __dividers = {
        "plus"        : '+',
        "minus"       : '-',
        "mul"         : '*',
        "div"         : '/',
        "mod"         : '%',
        "dot"         : '.',
        "or"          : "||",
        "and"         : "&&",
        "strict_eq"   : '===',
        "eq"          : '==',
        "strict_ne"   : '!==',
        "ne"          : '!=',
        "lsh"         : '<<',
        "le"          : '<=',
        "lt"          : '<',
        "ursh"        : '>>>',
        "rsh"         : '>>',
        "ge"          : '>=',
        "gt"          : '>',
        "bitwise_or"  : '|',
        "bitwise_xor" : '^',
        "bitwise_and" : '&'
    }

    __prefixes = {
        "increment"   : "++",
        "decrement"   : "--",
        "bitwise_not" : '~',
        "not"         : "!",
        "unary_plus"  : "+",
        "unary_minus" : "-",
        "delete"      : "delete ",
        "new"         : "new ",
        "typeof"      : "typeof ",
        "void"        : "void "
    }

    #
    # Script Scope
    #

    def type_script(self, node):
        return self.__statements(node)

    #
    # Expressions
    #

    def type_comma(self, node):
        return self.__commaSymbol.join(map(self.compress, node))

    def type_object_init(self, node):
        return "{%s}" % self.__commaSymbol.join(map(self.compress, node))

    def type_property_init(self, node):
        key = self.compress(node[0])
        value = self.compress(node[1])

        if type(key) in (int, float):
            pass

        elif self.__number_property.match(key):
            pass

        # Protect keywords and special characters
        elif key in keywords or key in futureReserved or not self.__simple_property.match(key):
            key = self.type_string(node[0])

        return "%s:%s" % (key, value)

    def type_array_init(self, node):
        def helper(child):
            return self.compress(child) if child != None else ""

        return "[%s]" % ",".join(map(helper, node))

    def type_array_comp(self, node):
        return "[%s %s]" % (self.compress(node.expression), self.compress(node.tail))

    def type_string(self, node):
        # Omit writing real high unicode character which are not supported well by browsers
        ascii = ascii_encoder.encode(node.value)

        if high_unicode.search(ascii):
            return ascii
        else:
            return unicode_encoder.encode(node.value)

    def type_number(self, node):
        value = node.value

        # Special handling for protected float/exponential
        if type(value) == str:
            # Convert zero-prefix
            if value.startswith("0.") and len(value) > 2:
                value = value[1:]

            # Convert zero postfix
            elif value.endswith(".0"):
                value = value[:-2]

        elif int(value) == value and node.parent.type != "dot":
            value = int(value)

        return "%s" % value

    def type_regexp(self, node):
        return node.value

    def type_identifier(self, node):
        return node.value

    def type_list(self, node):
        return ",".join(map(self.compress, node))

    def type_index(self, node):
        return "%s[%s]" % (self.compress(node[0]), self.compress(node[1]))

    def type_declaration(self, node):
        names = getattr(node, "names", None)
        if names:
            result = self.compress(names)
        else:
            result = node.name

        initializer = getattr(node, "initializer", None)
        if initializer:
            result += "=%s" % self.compress(node.initializer)

        return result

    def type_assign(self, node):
        assignOp = getattr(node, "assignOp", None)
        operator = "=" if not assignOp else self.__dividers[assignOp] + "="

        return self.compress(node[0]) + operator + self.compress(node[1])

    def type_call(self, node):
        return "%s(%s)" % (self.compress(node[0]), self.compress(node[1]))

    def type_new_with_args(self, node):
        result = "new %s" % self.compress(node[0])

        # Compress new Object(); => new Object;
        if len(node[1]) > 0:
            result += "(%s)" % self.compress(node[1])
        else:
            parent = getattr(node, "parent", None)
            if parent and parent.type is "dot":
                result += "()"

        return result

    def type_exception(self, node):
        return node.value

    def type_generator(self, node):
        """ Generator Expression """
        result = self.compress(getattr(node, "expression"))
        tail = getattr(node, "tail", None)
        if tail:
            result += " %s" % self.compress(tail)

        return result

    def type_comp_tail(self, node):
        """  Comprehensions Tails """
        result = self.compress(getattr(node, "for"))
        guard = getattr(node, "guard", None)
        if guard:
            result += "if(%s)" % self.compress(guard)

        return result

    def type_in(self, node):
        first = self.compress(node[0])
        second = self.compress(node[1])

        if first.endswith("'") or first.endswith('"'):
            pattern = "%sin %s"
        else:
            pattern = "%s in %s"

        return pattern % (first, second)

    def type_instanceof(self, node):
        first = self.compress(node[0])
        second = self.compress(node[1])

        return "%s instanceof %s" % (first, second)

    #
    # Statements :: Core
    #

    def type_block(self, node):
        return "{%s}" % self.__removeSemicolon(self.__statements(node))

    def type_let_block(self, node):
        begin = "let(%s)" % ",".join(map(self.compress, node.variables))
        if hasattr(node, "block"):
            end = self.compress(node.block)
        elif hasattr(node, "expression"):
            end = self.compress(node.expression)

        return begin + end

    def type_const(self, node):
        return self.__addSemicolon("const %s" % self.type_list(node))

    def type_var(self, node):
        return self.__addSemicolon("var %s" % self.type_list(node))

    def type_let(self, node):
        return self.__addSemicolon("let %s" % self.type_list(node))

    def type_semicolon(self, node):
        expression = getattr(node, "expression", None)
        return self.__addSemicolon(self.compress(expression) if expression else "")

    def type_label(self, node):
        return self.__addSemicolon("%s:%s" % (node.label, self.compress(node.statement)))

    def type_break(self, node):
        return self.__addSemicolon("break" if not hasattr(node, "label") else "break %s" % node.label)

    def type_continue(self, node):
        return self.__addSemicolon("continue" if not hasattr(node, "label") else "continue %s" % node.label)

    #
    # Statements :: Functions
    #

    def type_function(self, node):
        if node.type == "setter":
            result = "set"
        elif node.type == "getter":
            result = "get"
        else:
            result = "function"

        name = getattr(node, "name", None)
        if name:
            result += " %s" % name

        params = getattr(node, "params", None)
        result += "(%s)" % self.compress(params) if params else "()"

        # keep expression closure format (may be micro-optimized for other code, too)
        if getattr(node, "expressionClosure", False):
            result += self.compress(node.body)
        else:
            result += "{%s}" % self.__removeSemicolon(self.compress(node.body))

        return result

    def type_getter(self, node):
        return self.type_function(node)

    def type_setter(self, node):
        return self.type_function(node)

    def type_return(self, node):
        result = "return"
        if hasattr(node, "value"):
            valueCode = self.compress(node.value)

            # Micro optimization: Don't need a space when a block/map/array/group/strings are returned
            if not valueCode.startswith(("(","[","{","'",'"',"!","-","/")):
                result += " "

            result += valueCode

        return self.__addSemicolon(result)

    #
    # Statements :: Exception Handling
    #

    def type_throw(self, node):
        return self.__addSemicolon("throw %s" % self.compress(node.exception))

    def type_try(self, node):
        result = "try%s" % self.compress(node.tryBlock)

        for catch in node:
            if catch.type == "catch":
                if hasattr(catch, "guard"):
                    result += "catch(%s if %s)%s" % (self.compress(catch.exception), self.compress(catch.guard), self.compress(catch.block))
                else:
                    result += "catch(%s)%s" % (self.compress(catch.exception), self.compress(catch.block))

        if hasattr(node, "finallyBlock"):
            result += "finally%s" % self.compress(node.finallyBlock)

        return result

    #
    # Statements :: Loops
    #

    def type_while(self, node):
        result = "while(%s)%s" % (self.compress(node.condition), self.compress(node.body))
        self.__handleForcedSemicolon(node.body)
        return result

    def type_do(self, node):
        # block unwrapping don't help to reduce size on this loop type
        # but if it happens (don't like to modify a global function to fix a local issue), we
        # need to fix the body and re-add braces around the statement
        body = self.compress(node.body)
        if not body.startswith("{"):
            body = "{%s}" % body

        return self.__addSemicolon("do%swhile(%s)" % (body, self.compress(node.condition)))

    def type_for_in(self, node):
        # Optional variable declarations
        varDecl = getattr(node, "varDecl", None)

        # Body is optional - at least in comprehensions tails
        body = getattr(node, "body", None)
        if body:
            body = self.compress(body)
        else:
            body = ""

        result = "for"
        if node.isEach:
            result += " each"

        result += "(%s in %s)%s" % (self.__removeSemicolon(self.compress(node.iterator)), self.compress(node.object), body)

        if body:
            self.__handleForcedSemicolon(node.body)

        return result

    def type_for(self, node):
        setup = getattr(node, "setup", None)
        condition = getattr(node, "condition", None)
        update = getattr(node, "update", None)

        result = "for("
        result += self.__addSemicolon(self.compress(setup) if setup else "")
        result += self.__addSemicolon(self.compress(condition) if condition else "")
        result += self.compress(update) if update else ""
        result += ")%s" % self.compress(node.body)

        self.__handleForcedSemicolon(node.body)
        return result

    #
    # Statements :: Conditionals
    #

    def type_hook(self, node):
        """aka ternary operator"""
        condition = node.condition
        thenPart = node.thenPart
        elsePart = node.elsePart

        if condition.type == "not":
            [thenPart,elsePart] = [elsePart,thenPart]
            condition = condition[0]

        return "%s?%s:%s" % (self.compress(condition), self.compress(thenPart), self.compress(elsePart))

    def type_if(self, node):
        result = "if(%s)%s" % (self.compress(node.condition), self.compress(node.thenPart))

        elsePart = getattr(node, "elsePart", None)
        if elsePart:
            result += "else"

            elseCode = self.compress(elsePart)

            # Micro optimization: Don't need a space when the child is a block
            # At this time the brace could not be part of a map declaration (would be a syntax error)
            if not elseCode.startswith(("{", "(", ";")):
                result += " "

            result += elseCode

            self.__handleForcedSemicolon(elsePart)

        return result

    def type_switch(self, node):
        result = "switch(%s){" % self.compress(node.discriminant)
        for case in node:
            if case.type == "case":
                labelCode = self.compress(case.label)
                if labelCode.startswith('"'):
                    result += "case%s:" % labelCode
                else:
                    result += "case %s:" % labelCode
            elif case.type == "default":
                result += "default:"
            else:
                continue

            for statement in case.statements:
                temp = self.compress(statement)
                if len(temp) > 0:
                    result += self.__addSemicolon(temp)

        return "%s}" % self.__removeSemicolon(result)


import os
from zipfile import ZipFile

import pandas as pd
import streamlit as st
from PIL import Image

from app.inference_functions import feature_extraction, classification

classifier_file = 'app/classifiers/classifier_random_for.sav'
word2vec_file = 'app/word2vec/newsdata_model2'

# create US counties dataframe to allow people to choose counties
df = pd.read_csv('data/united-states-counties.csv')

def main():
    """Calls run function only if main"""
    image = Image.open('static/coronavirus.png')
    st.image(image, width=100, format='PNG')
    with open("app/intro.md", 'r') as file:
        st.markdown(file.read())
    run()

def run():
    """Function runs web-app"""
    # ask for input on activity and county location
    act_input = st.text_input("Please indicate the activity you would like to partake in.")
    states = sorted(df['State'].value_counts().index.tolist())
    state = st.selectbox('Choose State', states)
    count_input = None

    if state:
        counties = sorted(df[df['State'] == state]['County'].value_counts().index.tolist())
        count_input = st.selectbox('Choose County', counties)

    if act_input and count_input:
        if st.button('Check Risk'):

            count_input = count_input.replace(' County', '')
            features, feature_list, date, county_full = feature_extraction(act_input, count_input, word2vec_file)

            if features is None:
                st.markdown('No information about inputted activity, please try with another activity.')

            else:
                display_features = pd.DataFrame(features[100:-1].reshape(1, -1), columns=feature_list)

                # display results
                st.table(display_features)
                risk_level = classification(act_input, features, feature_list, date, county_full, classifier_file)
                st.markdown('OVERALL RISK LEVEL: ' + risk_level.upper())

# only run if main
if __name__ == "__main__":
    main()


import pickle
import sklearn

def readFile(file):
	return pickle.load(open(file, 'rb'))

def createLists(documents):
	tweets = []
	categories = []
	for category, tweet in documents:
		categories.append(category)
		# if "…" in tweet:
		# 	tweet = tweet.replace("…","...")
		tweets.append(tweet)

	return tweets, categories

def main():

	chosenFeatrue = "max" #"sum"

	svmfile = readFile('........pickle')
	kerasfile = readFile('........pickle')

	SVMemoji = [line[0] for line in open(svmfile,"r+").read().split("\n")]
	SVMprob  = [line[1] for line in open(svmfile,"r+").read().split("\n")]

	KERASemoji = [line[0] for line in open(kerasfile,"r+").read().split("\n")]
	KERASprob  = [line[1] for line in open(kerasfile,"r+").read().split("\n")]

	guess_categories = []

	with open("english.output.txt") as finalFile:
		for idx, emoji in enumerate(SVMemoji):
			if SVMemoji[idx] == KERASemoji[idx]:
				finalFile.write(SVMemoji[idx]+"\n")
				guess_categories.append(SVMemoji[idx])

			else:
				if chosenFeatrue == "max":
					if max(SVMprob[idx]) >= max(KERASprob[idx]):
						finalFile.write(SVMemoji[idx]+"\n")
						guess_categories.append(SVMemoji[idx])
					else:
						finalFile.write(KERASemoji[idx]+"\n")
						guess_categories.append(KERASemoji[idx])

				else:
					sumList = []
					for i, elem in enumerate(SVMprob[idx]):
						total = SVMprob[idx][i] + KERASprob[idx][i]
						sumList.append(total)

					emojiNR = sumList.index(max(sumList))
					finalFile.write(emojiNR+"\n")
					guess_categories.append(emojiNR[idx])

	test_documents = readFile('es-trial.pickle')
	test_tweets, test_categories = createLists(test_documents)

	#average scores
	precisionScoreAverage = sklearn.metrics.precision_score(test_categories,guess_categories, average="macro")
	print("\n\nprecision sklearn:", round(precisionScoreAverage,3))

	recallScoreAverage = sklearn.metrics.recall_score(test_categories,guess_categories, average="macro")
	print("recall sklearn:", round(recallScoreAverage,3))

	f1ScoreAverage = sklearn.metrics.f1_score(test_categories,guess_categories, average="macro")
	print("fscore sklearn:", round(f1ScoreAverage,3),"\n")

	#scores per class
	labels = ["0","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19"]
	print("Class \t Precision \t Recall \t F-score")
	for label in labels:
		precisionScore = sklearn.metrics.precision_score(test_categories,guess_categories, average="macro", labels=label)
		recallScore = sklearn.metrics.recall_score(test_categories,guess_categories, average="macro", labels=label)
		f1Score = sklearn.metrics.f1_score(test_categories,guess_categories, average="macro", labels=label)

		print(label, "\t", round(precisionScore,3), "\t\t", round(recallScore,3), "\t\t", round(f1Score,3), "\t")

	print()

if __name__ == '__main__':
	main()


import os
from io import BytesIO
import allura
import json

import PIL
from ming.odm.odmsession import ThreadLocalODMSession
from mock import patch
from tg import config

from allura import model as M
from allura.lib import helpers as h
from allura.tests import decorators as td
from alluratest.controller import TestController

from forgewiki import model
from unittest.mock import MagicMock

class TestRootController(TestController):

    def setup_method(self, method):
        super().setup_method(method)
        self.setup_with_tools()

    @td.with_wiki
    def setup_with_tools(self):
        pass

    def _find_edit_form(self, resp):
        def cond(f):
            return f.id == 'page_edit_form'
        return self.find_form(resp, cond)

    def test_root_index(self):
        page_url = h.urlquote('/wiki/tést/')
        r = self.app.get(page_url).follow()
        assert 'tést' in r
        assert 'Create Page' in r
        # No 'Create Page' button if user doesn't have 'create' perm
        r = self.app.get('/wiki/Home',
                         extra_environ=dict(username='*anonymous'))
        assert 'Create Page' not in r, r

    def test_create_wiki_page(self):
        url = "/p/test/wiki/create_wiki_page/"
        r = self.app.get(url)
        assert 'test' in r
        assert 'Create page' in r.text

    def test_root_markdown_syntax(self):
        response = self.app.get('/wiki/markdown_syntax/', status=301)
        assert response.location.endswith('/nf/markdown_syntax')

    def test_root_browse_tags(self):
        response = self.app.get('/wiki/browse_tags/')
        assert 'Browse Labels' in response

    def test_root_browse_pages(self):
        response = self.app.get('/wiki/browse_pages/')
        assert 'Browse Pages' in response

    def test_root_new_page(self):
        response = self.app.get('/wiki/new_page?title=' + h.urlquote('tést'))
        assert response.location == 'http://localhost/wiki/t%C3%A9st/'

    def test_root_new_search(self):
        self.app.get(h.urlquote('/wiki/tést/'))
        response = self.app.get('/wiki/search/?q=' + h.urlquote('tést'))
        assert 'Search wiki: tést' in response

    def test_feed(self):
        for ext in ['', '.rss', '.atom']:
            self.app.get('/wiki/feed%s' % ext, status=200)

    @patch('allura.lib.helpers.ceil',  MagicMock(return_value=1))
    @patch('allura.lib.search.search')
    def test_search(self, search):
        r = self.app.get('/wiki/search/?q=test')
        assert (
            'relevance' in r)
        assert (
            'date' in r)

        p = M.Project.query.get(shortname='test')
        r = self.app.get('/wiki/search/?q=test&sort=score+asc')
        solr_query = {
            'short_timeout': True,
            'ignore_errors': False,
            'rows': 25,
            'start': 0,
            'qt': 'dismax',
            'qf': 'title^2 text',
            'pf': 'title^2 text',
            'fq': [
                'project_id_s:%s' % p._id,
                'mount_point_s:wiki',
                '-deleted_b:true',
                'type_s:("WikiPage" OR "WikiPage Snapshot")',
                'is_history_b:False',
            ],
            'hl': 'true',
            'hl.simple.pre': '#ALLURA-HIGHLIGHT-START#',
            'hl.simple.post': '#ALLURA-HIGHLIGHT-END#',
            'sort': 'score asc',
        }
        search.assert_called_with('test', **solr_query)

        r = self.app.get(
            '/wiki/search/?q=test&search_comments=on&history=on&sort=mod_date_dt+desc')
        solr_query['fq'][
            3] = 'type_s:("WikiPage" OR "WikiPage Snapshot" OR "Post")'
        solr_query['fq'].remove('is_history_b:False')
        solr_query['sort'] = 'mod_date_dt desc'
        search.assert_called_with('test', **solr_query)

        r = self.app.get('/wiki/search/?q=test&parser=standard')
        solr_query['sort'] = 'score desc'
        solr_query['fq'][3] = 'type_s:("WikiPage" OR "WikiPage Snapshot")'
        solr_query['fq'].append('is_history_b:False')
        solr_query.pop('qt')
        solr_query.pop('qf')
        solr_query.pop('pf')
        search.assert_called_with('test', **solr_query)

    def test_search_help(self):
        r = self.app.get('/wiki/search/?q=test')
        btn = r.html.find('a', attrs={'class': 'icon btn search_help_modal'})
        assert btn is not None, "Can't find a help button"
        div = r.html.find('div', attrs={'id': 'lightbox_search_help_modal'})
        assert div is not None, "Can't find help text"
        assert 'To search for an exact phrase' in div.text

    def test_nonexistent_page_edit(self):
        resp = self.app.get(h.urlquote('/wiki/tést/'))
        assert resp.location.endswith(h.urlquote('/wiki/tést/edit')), resp.location
        resp = resp.follow()
        assert 'tést' in resp

    def test_nonexistent_page_noedit(self):
        self.app.get(h.urlquote('/wiki/tést/'),
                     extra_environ=dict(username='*anonymous'),
                     status=404)
        self.app.get(h.urlquote('/wiki/tést/'),
                     extra_environ=dict(username='test-user'),
                     status=404)

    @patch('forgewiki.wiki_main.g.director.create_activity')
    def test_activity(self, create_activity):
        d = dict(title='foo', text='footext')
        self.app.post('/wiki/foo/update', params=d)
        assert create_activity.call_count == 1
        assert create_activity.call_args[0][1] == 'created'
        create_activity.reset_mock()
        d = dict(title='foo', text='new footext')
        self.app.post('/wiki/foo/update', params=d)
        assert create_activity.call_count == 1
        assert create_activity.call_args[0][1] == 'modified'
        create_activity.reset_mock()
        d = dict(title='new foo', text='footext')
        self.app.post('/wiki/foo/update', params=d)
        assert create_activity.call_count == 1
        assert create_activity.call_args[0][1] == 'renamed'

    def test_labels(self):
        response = self.app.post(
            '/wiki/foo-bar/update',
            params={
                'title': 'foo',
                'text': 'sometext',
                'labels': 'test label',
            }).follow()
        assert ('test label (1)' in
                response)

    def test_title_slashes(self):
        # forward slash not allowed in wiki page title - converted to dash
        response = self.app.post(
            '/wiki/foo-bar/update',
            params={
                'title': 'foo/bar',
                'text': 'sometext',
                'labels': '',
            }).follow()
        assert 'foo-bar' in response
        assert 'foo-bar' in response.request.url

    def test_dotted_page_name(self):
        r = self.app.post(
            '/wiki/page.dot/update',
            params={
                'title': 'page.dot',
                'text': 'text1',
                'labels': '',
            }).follow()
        assert 'page.dot' in r

    def test_subpage_attempt(self):
        self.app.get(h.urlquote('/wiki/tést/'))
        self.app.post(
            h.urlquote('/wiki/tést/update'),
            params={
                'title': 'tést'.encode(),
                'text': 'text1',
                'labels': '',
            })
        assert '/p/test/wiki/Home/' in self.app.get(h.urlquote('/wiki/tést/Home/'))
        self.app.get(h.urlquote('/wiki/tést/notthere/'), status=404)

    def test_page_history(self):
        self.app.get(h.urlquote('/wiki/tést/'))
        self.app.post(
            h.urlquote('/wiki/tést/update'),
            params={
                'title': 'tést'.encode(),
                'text': 'text1',
                'labels': '',
            })
        self.app.post(
            h.urlquote('/wiki/tést/update'),
            params={
                'title': 'tést'.encode(),
                'text': 'text2',
                'labels': '',
            })
        response = self.app.get(h.urlquote('/wiki/tést/history'))
        assert 'tést' in response
        # two revisions are shown
        assert '
' in response
        assert '
' in response
        # you can revert to an old revison, but not the current one
        assert response.html.find('a', {'data-dialog-id': '1'}), response.html
        assert not response.html.find('a', {'data-dialog-id': '2'})
        response = self.app.get(h.urlquote('/wiki/tést/history'),
                                extra_environ=dict(username='*anonymous'))
        # two revisions are shown
        assert '
' in response
        assert '
' in response
        # you cannot revert to any revision
        assert not response.html.find('a', {'data-dialog-id': '1'})
        assert not response.html.find('a', {'data-dialog-id': '2'})

        # view an older version
        response = self.app.get(h.urlquote('/wiki/tést/') + '?version=1')
        response.mustcontain('text1')
        response.mustcontain(no='text2')

    def test_page_diff(self):
        self.app.post(
            h.urlquote('/wiki/tést/update'),
            params={
                'title': 'tést'.encode(),
                'text': 'sometext',
                'labels': '',
            })
        self.app.post(h.urlquote('/wiki/tést/revert'), params=dict(version='1'))
        response = self.app.get(h.urlquote('/wiki/tést/diff') + '?v1=0&v2=0')
        assert 'tést' in response
        d = dict(title='testdiff', text="""**Optionally**, you may also want to remove all the unused accounts that have accumulated (one was created for *every* logged in SF-user who has visited your MediaWiki hosted app):

                                            ~~~~~
                                            php removeUnusedAccounts.php --delete
                                            ~~~~~

                                            #### 6) Import image (and other) files into your Mediawiki install ####

                                            Upload the backup of your data files to the project web.

                                            ~~~~~
                                            scp projectname_mediawiki_files.tar.gz USERNAME@web.domain.net:
                                            ~~~~~

                                            In the project web shell, unpack the files to the images directory of you wiki installation. In the backup, the images are in a subfolder *projectname*, so follow these steps:

                                            ~~~~~
                                            cd wiki
                                            mkdir oldimages
                                            cd oldimages
                                            tar -xvzf ../../../projectname_mediawiki_files.tar.gz
                                            mv projectname/* ../images/
                                            cd ..
                                            rm -r oldimages
                                            # Now fix permissons. Wrong permissions may cause massive slowdown!
                                            chown yournick:apache images/ --recursive
                                            chmod 775 images/ --recursive
                                            ~~~~~

                                            **TODO: FIXME:** The following can't be quite correct:

                                            Now hit your wiki a few times from a browser. Initially, it will be dead slow, as it is trying to build thumbnails for the images. And it will time out, a lot. Keep hitting reload, until it works.

                                            **Note:** The logo shown in the sidebar is no longer stored as an object in the wiki (as it was in the Hosted App installation). Rather save it as a regular file, then edit LocalSettings.php, adding""")
        self.app.post('/wiki/testdiff/update', params=d)
        d = dict(title='testdiff', text="""**Optionally**, you may also want to remove all the unused accounts that have accumulated (one was created for *every* logged in SF-user who has visited your MediaWiki hosted app):

                                            ~~~~~
                                            php removeUnusedAccounts.php --delete
                                            ~~~~~

                                            #### 6) Import image (and other) files into your Mediawiki install ####

                                            Upload the backup of your data files to the project web.

                                            ~~~~~
                                            scp projectname_mediawiki_files.tar.gz USERNAME@web.domain.net:
                                            ~~~~~

                                            In the project web shell, unpack the files to the images directory of you wiki installation. In the backup, the images are in a subfolder *projectname*, so follow these steps:

                                            ~~~~~
                                            cd wiki
                                            mkdir oldimages
                                            cd oldimages
                                            tar -xvzf ../../../projectname_mediawiki_files.tar.gz
                                            mv projectname/* ../images/
                                            cd ..
                                            rm -r oldimages
                                            # Now fix permissions. Wrong permissions may cause a massive slowdown!
                                            chown yournick:apache images/ --recursive
                                            chmod 775 images/ --recursive
                                            ~~~~~

                                            **TODO: FIXME:** The following can't be quite correct:

                                            Now hit your wiki a few times from a browser. Initially, it will be dead slow, as it is trying to build thumbnails for the images. And it will time out, a lot. Keep hitting reload, until it works.

                                            **Note:** The logo shown in the sidebar is no longer stored as an object in the wiki (as it was in the Hosted App installation). Rather save it as a regular file, then edit LocalSettings.php, adding

                                            """)
        self.app.post('/wiki/testdiff/update', params=d)
        response = self.app.get('/wiki/testdiff/diff?v1=1&v2=2')
        assert ('# Now fix  permissons.   permissions.  '
                'Wrong permissions may cause  a  massive slowdown!' in
                response)
        assert '


# libraries
import numpy as np
import matplotlib.pyplot as plt
#plt.style.use(‘seaborn-whitegrid’)
# width of the bars
barWidth = 0.3
plt.figure(figsize=(10,3))
# Choose the height of the blue bars
bars1 = [0.732, 0.845, 0.863, 0.876, 0.887]
# Choose the height of the cyan bars
bars2 = [0.803, 0.916, 0.926, 0.934, 0.935]
bars3 = [0.886, 0.931, 0.941, 0.947, 0.949]
bars4 = [0.897, 0.937, 0.944, 0.952, 0.956]
bars5 = [0.906, 0.938, 0.949, 0.953, 0.954]
# Choose the height of the error bars (bars1)
yer1 = [0.02, 0.01, 0.005, 0.01, 0.015]
yer2 = [0.015, 0.013, 0.005, 0.006, 0.01]
yer3 = [0.003, 0.01, 0.005, 0.004, 0.003]
yer4 = [0.02, 0.01, 0.005, 0.004, 0.003]
yer5 = [0.02, 0.01, 0.005, 0.004, 0.003]
# The x position of bars
r1 = np.array([0,2,4,6,8])
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
r4 = [x + barWidth for x in r3]
r5 = [x + barWidth for x in r4]
# Create blue bars
plt.bar(r1, bars1, width=barWidth, alpha=0.5, color ='C0', edgecolor='black', yerr=yer1, capsize=7, label='CE')
plt.yticks(np.arange(0, 1, 0.03))
# Create cyan bars
plt.bar(r2, bars2, width=barWidth, alpha=0.5, color = 'C1', edgecolor='black', yerr=yer2, capsize=7, label='FL')
plt.bar(r3, bars3, width=barWidth, alpha=0.5, color = 'C2', edgecolor='black', yerr=yer3, capsize=7, label='FL(random)')
plt.bar(r4, bars4, width=barWidth, alpha=0.5, color = 'C3', edgecolor='black', yerr=yer4, capsize=7, label='FL(feature)')
plt.bar(r5, bars5, width=barWidth, alpha=0.5, color = 'C4', edgecolor='black', yerr=yer5, capsize=7, label='FL(attribute)')
# general layout
plt.xticks([r + 2*barWidth for r in r1], ['399/23%', '999/23%', '1999/23%','2999/23%','3999/23%'])
plt.ylabel('AUROC')
plt.xlabel('size/positive label percentage')
plt.ylim(0.65, 1)
#plt.legend(loc = “lower right”)
plt.legend(loc=2, bbox_to_anchor=(1.05,1.0),borderaxespad = 0.)
# Show graphic
#plt.show()
#plt.savefig(“output.pdf”, format=‘pdf’,bbox_inches = ‘tight’)


import json
import os
from time import sleep

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.webdriver import WebDriver

class BasePage():
    _url = ''
    def __init__(self,driver_basepage:WebDriver = None):

        if driver_basepage == None:
            self.driver = webdriver.Chrome()
        else:
            self.driver = driver_basepage
        if self._url != '':
            self.driver.get(self._url)

        self.driver.implicitly_wait(10)

    def find(self,by,location):
        return self.driver.find_element(by,location)

    def get_cookies(self):
        cookies = self.driver.get_cookies()
        with open("cookies.json", 'w') as f:
            json.dump(cookies, f)

    def add_cookie(self):
        with open("cookies.json",'r')as f:
            cookies = json.load(f)
            print(cookies)
            for cookie in cookies:
                if 'expiry' in cookie.keys():
                    del cookie['expiry']
                self.driver.add_cookie(cookie)
            sleep(20)
            self.driver.refresh()


# (c) Nelen & Schuurmans.  GPL licensed, see LICENSE.rst.
# -*- coding: utf-8 -*-

"""Implementation of the Project archive task."""

# Python 3 is coming
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import logging

from .models import Project, Measurement

logger = logging.getLogger(__name__)

def archive(project_id):
    """Archive a project.

    This has a side effect for Projects with measurements with measurement
    types that have the delete_on_archive field set: those measurements
    will be deleted, both in db and on disk. The motivation for this is
    that we want to remove 'attachment' measurement files, e.g., all media
    files belonging to a ribx, but not the ribx itself.
    """
    try:
        project = Project.objects.get(pk=project_id)
    except Project.DoesNotExist:
        logger.warn("project_id %s not found in task", project_id)
        return
    logger.info("Archiving project %s", project)
    # This query returns all Measurements that (1) belong to a Project,
    # (2) that have a parent Measurement, which entails that it is an
    # attachment (e.g., media files belonging to a Ribx), and (3) have
    # a measurement type that can be deleted.
    measurements = Measurement.objects.filter(
        location__activity__project=project,
        parent__isnull=False,
        location__activity__measurement_type__delete_on_archive=True)
    logger.debug("Deleting measurements: %s", measurements)
    for m in measurements:
        m.delete(notify=False, deleted_by_contractor=False,
                 set_completeness=False)
    project.is_archived = True
    project.save()


import os

from gym import error, spaces
from gym.utils import seeding
import numpy as np
from os import path
import gym
import six
import pygame as pg
from pygame.locals import *
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *
try:
    import bindings as pyFlex
except ImportError as e:
    raise error.DependencyNotInstalled("{}. (HINT: PyFlex Binding is not installed correctly)".format(e))

class FlexEnv(gym.Env):
    def __init__(self, frame_skip, observation_size, observation_bounds,action_bounds,
                 dt=1 / 60.0, obs_type="parameter", action_type="continuous", scene=0, viewer=1):
        assert obs_type in ('parameter', 'image')
        assert action_type in ("continuous", "discrete")

        self.viewerId = viewer
        self.screen = None
        self.screen_size = (800,800)
        self.sub_screen_gap = 4

        flex_viewer = False
        if self.viewerId == 1:
            flex_viewer = True
        elif self.viewerId == 2:
            pg.init()
            # self.screen = pg.display.set_mode(self.screen_size)
            self.screen = pg.display.set_mode(self.screen_size,DOUBLEBUF|OPENGL)

            gluOrtho2D(0,self.screen_size[0],self.screen_size[1],0)
            # gluPerspective(45, (self.screen_size[0] / self.screen_size[1]), 0.1, 50.0)
        elif self.viewerId == 3:
            flex_viewer = True
            self.screen = pg.display.set_mode(self.screen_size)

        pyFlex.setVisualize(flex_viewer)
        pyFlex.chooseScene(scene)
        pyFlex.initialize()

        pyFlex.setDt(dt)
        self.dt = dt
        print('pyFlex initialization OK')

        self._obs_type = obs_type
        self.frame_skip = frame_skip
        self.numInstances = pyFlex.getNumInstances()

        # assert not done
        self.obs_dim = observation_size
        self.act_dim = len(action_bounds[0])
        # for discrete instances, action_space should be defined in the subclass
        self.action_space = spaces.Box(action_bounds[0], action_bounds[1])
        self.observation_space = spaces.Box(observation_bounds[0], observation_bounds[1])

        self.metadata = {
            'video.frames_per_second': int(np.round(1.0 / self.dt))
        }

        self.save_video = False

        self.video_path = "/home/yzhang/data"
        # self.video_path = "/home/dragonmyth/data"

        self.step_cnt = 0

    def _seed(self, seed=None):
        if seed is None:
            pyFlex.setSceneRandSeed(-1)
        else:
            np.random.seed(seed)
            pyFlex.setSceneRandSeed(seed)
        # self.np_random, sseed = seeding.np_random(seed)

        return [seed]

    def do_simulation(self, tau, n_frames):
        self.step_cnt += 1

        done = False
        save = self.save_video
        path = os.path.join(self.video_path, 'frame_%d.tga' % self.step_cnt)

        pyFlex.simulateKSteps(save,path,tau.flatten(),n_frames)
        # for _ in range(n_frames):
        #     path = os.path.join(self.video_path, 'frame_%d.tga' % self.step_cnt)
        #     pyFlex.update_frame(save,path,tau.flatten())
        #     save = False
        #     done=pyFlex.sdl_main()
        #     if done:
        #         break

        return done
    def _step(self, action):
        done = self.do_simulation(action,self.frame_skip)
        obs = self.get_state()
        reward = 0
        info = {}
        return obs,reward,done,info

    def _reset(self):
        # print("ASFASFASFASF")
        # self._seed(self.seed)
        state = pyFlex.resetScene()
        palceholder_act = np.zeros((self.numInstances,7))
        palceholder_act[:,-1] = 1
        self.do_simulation(palceholder_act, 200)

        return state
    def get_state(self):
        state_vec = pyFlex.get_state()
        # part_state = state_vec[:-4*self.numInstances]
        # bar_state = state_vec[-4*self.numInstances::]
        #
        # part_state = part_state.reshape([self.numInstances,int(part_state.shape[0]/self.numInstances),2])
        # bar_state = bar_state.reshape([self.numInstances,int(bar_state.shape[0]/self.numInstances),2])
        #
        # full_state = np.concatenate([bar_state,part_state],axis=1)
        # return full_state
        full_state = state_vec.reshape([self.numInstances,int(state_vec.shape[0]/self.numInstances),3])
        return full_state

    def set_controller(self,controllerConfig):
        pyFlex.setController(controllerConfig)

    def set_goal(self,goals):
        pyFlex.setGoal(goals)
    def setMapHalfExtent(self,halfExtent):
        pyFlex.setMapHalfExtent(halfExtent)
    def setInitClusterParam(self,clusterParam):
        pyFlex.setInitClusterParam(clusterParam)
    def get_density(self,particles,resolution,width,mapHalfExtent):
        return pyFlex.getParticleDensity(particles, resolution,width,mapHalfExtent)
    def get_height_map(self,particles,heights,resolution,width,mapHalfExtent):
        return pyFlex.getParticleHeightMap(particles,heights,resolution,width,mapHalfExtent)
    def get_angular_vel_flex(self,partPos,partVel):
        return pyFlex.getParticleAngularVelocity(partPos,partVel)
    def set_aux_info(self,auxInfo):
        return pyFlex.setAuxInfo(auxInfo)
    def _render(self, mode='human', close=False):
        if(self.viewerId==2):
            # pg.display.update()
            pg.display.flip()
            # img = pg.surfarray.array3d(self.screen).transpose([1, 0, 2])
            img = np.zeros((1,1,3))
        elif(self.viewerId==3):
            pg.display.update()
            img = pg.surfarray.array3d(self.screen).transpose([1, 0, 2])
        else:

            img = np.zeros((1,1,3))
        return img

if __name__ == '__main__':
    env = FlexEnv(frame_skip=5, observation_size=10, action_bounds=np.array([[-3, -3, -1, -1], [3, 3, 1, 1]]),scene=0)

    while True:
        pyFlex.resetScene()
        for _ in range(5000):
            # print(pyFlex.get_state())
            act = np.random.uniform([-2, -2, -1, -1]*env.numInstances, [2, 2, 1, 1]*env.numInstances,env.numInstances*4)
            obs,rwd,done,info = env.step(act)
            if done:
                break
        else:
            continue
        break


#!/usr/bin/env python

import numpy as np
import os
import shutil
import mss
import matplotlib
matplotlib.use('TkAgg')
from datetime import datetime
from matplotlib.figure import Figure
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg as FigCanvas
import cv2
from PIL import ImageTk, Image
import imutils
import sys
import pygame
import time

from tqdm import tqdm

PY3_OR_LATER = sys.version_info[0] >= 3

class PS4Controller:
    def __init__(self):
        pygame.init()
        for event in pygame.event.get():
            pass
        try:
            print("Name of the joystick:")

            self.joystick = pygame.joystick.Joystick(0)
            self.joystick.init()
            print(self.joystick.get_name())
        except:
            self.joystick = None

    def read(self):
        if not self.joystick:
            return [0,0,0,0,0]
        for event in pygame.event.get():
            pass
        x = self.joystick.get_axis(0)
        y = self.joystick.get_axis(1)
        a = self.joystick.get_button(0)
        b = self.joystick.get_button(1)
        rb = self.joystick.get_button(2)
        r = [x, y, a, b, rb]
        print(r)
        return r

if PY3_OR_LATER:
    # Python 3 specific definitions
    import tkinter as tk
    import tkinter.ttk as ttk
    import tkinter.messagebox as tkMessageBox
else:
    # Python 2 specific definitions
    import Tkinter as tk
    import ttk
    import tkMessageBox

from utils import Screenshot, XboxController, get_frame

IMAGE_SIZE = (320, 240)
IDLE_SAMPLE_RATE = 200
SAMPLE_RATE = 100

class MainWindow():
    """ Main frame of the application
    """

    def __init__(self):
        self.prev = 0
        self.rec = []
        self.cap = cv2.VideoCapture(0)
        self.root = tk.Tk()
        self.sct = mss.mss()

        self.root.title('Data Acquisition')
        self.root.geometry("660x325")
        self.root.resizable(False, False)

        # Init controller
        self.controller = PS4Controller()

         # Create GUI
        self.create_main_panel()

        # Timer
        self.rate = IDLE_SAMPLE_RATE
        self.sample_rate = SAMPLE_RATE
        self.idle_rate = IDLE_SAMPLE_RATE
        self.recording = False
        self.t = 0
        self.pause_timer = False
        self.on_timer()

        self.root.mainloop()

    def create_main_panel(self):
        # Panels
        top_half = tk.Frame(self.root)
        top_half.pack(side=tk.TOP, expand=True, padx=5, pady=5)
        message = tk.Label(self.root, text="(Note: UI updates are disabled while recording)")
        message.pack(side=tk.TOP, padx=5)
        bottom_half = tk.Frame(self.root)
        bottom_half.pack(side=tk.LEFT, padx=5, pady=10)

        # Images
        self.img_panel = tk.Label(top_half, image=ImageTk.PhotoImage("RGB", size=IMAGE_SIZE)) # Placeholder
        self.img_panel.pack(side = tk.LEFT, expand=False, padx=5)

        # Joystick
        self.init_plot()
        self.PlotCanvas = FigCanvas(figure=self.fig, master=top_half)
        self.PlotCanvas.get_tk_widget().pack(side=tk.RIGHT, expand=False, padx=5)

        # Recording
        textframe = tk.Frame(bottom_half, width=332, height=15, padx=5)
        textframe.pack(side=tk.LEFT)
        textframe.pack_propagate(0)
        self.outputDirStrVar = tk.StringVar()
        self.txt_outputDir = tk.Entry(textframe, textvariable=self.outputDirStrVar, width=100)
        self.txt_outputDir.pack(side=tk.LEFT)
        self.outputDirStrVar.set("samples/" + datetime.now().strftime('%Y-%m-%d_%H:%M:%S'))

        self.record_button = ttk.Button(bottom_half, text="Record", command=self.on_btn_record)
        self.record_button.pack(side = tk.LEFT, padx=5)

    def init_plot(self):
        self.plotMem = 50 # how much data to keep on the plot
        self.plotData = [[0] * (5)] * self.plotMem # mem storage for plot

        self.fig = Figure(figsize=(4,3), dpi=80) # 320,240
        self.axes = self.fig.add_subplot(111)

    def on_timer(self):
        now = time.time()

        print(f"Timer: {now} {now-self.prev}")
        self.prev = now
        if not self.pause_timer:
            self.root.after(self.rate, self.on_timer)

        self.poll()
        # stop drawing if recording to avoid slow downs
        if self.recording == False:
            self.draw()
        else:
            self.draw_plot()

    def poll(self):
        self.img = self.take_screenshot()
        self.controller_data = self.controller.read()
        self.update_plot()

        if self.recording == True:
            self.save_data()
            self.t += 1

    def take_screenshot(self):
        frame = get_frame(self.cap)
        return frame

    def update_plot(self):
        self.plotData.append(self.controller_data) # adds to the end of the list
        self.plotData.pop(0) # remove the first item in the list, ie the oldest

    def save_data(self):
        image_file = self.outputDir+'/'+'img_'+str(self.t)+'.png'
        cv2.imwrite(image_file, self.img)

        # write csv line
        self.outfile.write( image_file + ',' + ','.join(map(str, self.controller_data)) + '\n' )

    def draw(self):
        # Image
        frame = get_frame(self.cap)
        frame = frame[:,:,::-1]
        new_frame = imutils.resize(frame, width=300)
        image = Image.fromarray(new_frame, mode="RGB")
        self.img_panel.img = ImageTk.PhotoImage(image)
        self.img_panel['image'] = self.img_panel.img
        self.draw_plot()

    def draw_plot(self):
        # Joystick
        x = np.asarray(self.plotData)
        self.axes.clear()
        self.axes.plot(range(0,self.plotMem), x[:,0], 'r')
        self.axes.plot(range(0,self.plotMem), x[:,1], 'b')
        self.axes.plot(range(0,self.plotMem), x[:,2], 'g')
        self.axes.plot(range(0,self.plotMem), x[:,3], 'k')
        self.axes.plot(range(0,self.plotMem), x[:,4], 'y')
        self.PlotCanvas.draw()

    def on_btn_record(self):
        # pause timer

        if self.recording:
            self.recording = False
        else:
            self.start_recording()

        if self.recording:
            self.rec = []
            self.t = 0 # Reset our counter for the new recording
            self.record_button["text"] = "Stop"
            self.rate = self.sample_rate
            # make / open outfile
            self.outfile = open(self.outputDir+'/'+'data.csv', 'a')
        else:
            self.record_button["text"] = "Record"
            self.rate = self.idle_rate
            self.outfile.close()

        # un pause timer
        self.pause_timer = False
        # self.on_timer()

    def start_recording(self):
        should_record = True

        # check that a dir has been specified
        if not self.outputDirStrVar.get():
            tkMessageBox.showerror(title='Error', message='Specify the Output Directory', parent=self.root)
            should_record = False

        else: # a directory was specified
            self.outputDir = self.outputDirStrVar.get()

            # check if path exists - i.e. may be saving over data
            if os.path.exists(self.outputDir):

                # overwrite the data, yes/no?
                if tkMessageBox.askyesno(title='Warning!', message='Output Directory Exists - Overwrite Data?', parent=self.root):
                    # delete & re-make the dir:
                    shutil.rmtree(self.outputDir)
                    os.mkdir(self.outputDir)

                # answer was 'no', so do not overwrite the data
                else:
                    should_record = False
                    self.txt_outputDir.focus_set()

            # directory doesn't exist, so make one
            else:
                os.mkdir(self.outputDir)

        self.recording = should_record

if __name__ == '__main__':
    app = MainWindow()


import pandas as pd
import numpy as np
import scipy.stats as stats
import random

chat_id = 465374385 # Ваш chat ID, не меняйте название переменной

def solution(data) -> bool: # Одна или две выборке на входе, заполняется исходя из условия
    alpha = 0.06

# Пороговое значение
    threshold = 300

# Расчет среднего значения и стандартного отклонения
    mean = sum(data) / len(data)
    std = stats.tstd(data)

# Расчет t-статистики
    n = len(data)
    t_stat = (mean - threshold) / (std / (n**0.5))

# Расчет критического значения
    crit_val = stats.t.ppf(alpha, df=n-1)

# Принятие решения
    random_bool = random.choice([True, False])

# Выводим случайное значение
# Принятие решения
    return random_bool


from datetime import datetime, timedelta

def set_time(date_str:str) -> None:
    '''
    Make a datetime object from a string.
    '''

    try:
        date_obj = datetime.strptime(date_str, "%d/%m/%Y")
    except ValueError:
        print("Invalid date format, should be dd/mm/yyyy")
        return None
    print("Set_TIME")
    print(f"DateTime String:\t{date_str}")
    print(f"DateTime Object:\t{date_obj}")
    print(f"Date:\t{date_obj.date()}")
    print("\n\n\n")

def add_days(date_str:str, days:int) -> None:
    '''
    Add days to a date.
    '''

    try:
        date_obj = datetime.strptime(date_str, "%d/%m/%Y")
    except ValueError:
        print("Invalid date format, should be dd/mm/yyyy")
        return None

    date_obj += timedelta(days=days)
    print("ADD_DAYS")
    print(f"DateTime String:\t{date_str}\tOffset:\t{days}")
    print(f"DateTime Object:\t{date_obj}")
    print(f"Date:\t{date_obj.date()}")
    print("\n\n\n")

def main():
    set_time("07/06/2022")
    add_days("07/06/2022", 1)
    add_days("07/06/2022", -1)

if __name__ == "__main__":
    main()