"""Adds column to use scicrunch alternative

Revision ID: b60363fe438f
Revises: 39fa67f45cc0
Create Date: 2020-12-15 18:26:25.552123+00:00

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "b60363fe438f"
down_revision = "39fa67f45cc0"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column(
        "group_classifiers",
        sa.Column("uses_scicrunch", sa.Boolean(), nullable=True, server_default="0"),
    )
    # ### end Alembic commands ###

    # Applies the default to all
    query = 'UPDATE "group_classifiers" SET uses_scicrunch=false;'
    op.execute(query)

    # makes non nullable
    # 'ALTER TABLE "group_classifiers" ALTER "uses_scicrunch" SET NOT NULL;'
    op.alter_column("group_classifiers", "uses_scicrunch", nullable=False)

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column("group_classifiers", "uses_scicrunch")
    # ### end Alembic commands ###


import unittest
from unittest.mock import patch
import os
from typing import Optional
from dataclasses import dataclass
from io import StringIO

from ml_project.train_pipeline import run_train_pipeline
from ml_project.predict_pipeline import run_predict_pipeline
from sklearn.preprocessing import StandardScaler

from ml_project.entities import (
    TrainingPipelineParams,
    SplitParams,
    FeatureParams,
    TrainingParams
)

numerical_features = [
    "age",
    "sex",
    "cp",
    "trestbps",
    "chol",
    "fbs",
    "restecg",
    "thalach",
    "exang",
    "oldpeak",
    "slope",
    "ca",
    "thal",
]

@dataclass
class TestTrainingPipelineParams:
    input_data_path: str = "data/raw/heart_cleveland_upload.csv"
    output_model_path: str = "tests/tmp/test_model.pkl"
    metric_path: str = "tests/tmp/test_metrics.json"
    split_params: SplitParams = SplitParams(
        test_size=0.25,
        random_state=5
    )
    feature_params: FeatureParams = FeatureParams(
        numerical_features=numerical_features,
        target_col="condition"
    )
    train_params: TrainingParams = TrainingParams(
        model_type="RandomForestClassifier",
    )
    train_dataframe_path: Optional[str] = "data/raw/predict_dataset.csv"
    scaler: Optional[str] = None

@dataclass
class TestPredictPipelineParams:
    input_data_path: str = "data/raw/predict_dataset.csv"
    input_model_path: str = "models/model.pkl"
    output_data_path: str = "tests/tmp/test_model_predicts.csv"

class TestEnd2End(unittest.TestCase):
    test_train_piplein_params = TestTrainingPipelineParams()
    test_test_piplein_params = TestPredictPipelineParams()

    @unittest.mock.patch("ml_project.train_pipeline.logger")
    def test_train_end2end(self, mock_log):
        with patch("sys.stdout", new=StringIO()):
            path_to_model, metrics = run_train_pipeline(self.test_train_piplein_params)

            self.assertTrue(os.path.exists(path_to_model))
            self.assertTrue(metrics["0"]["f1-score"] > 0.6)
            self.assertTrue(metrics["1"]["f1-score"] > 0.6)

    @unittest.mock.patch("ml_project.train_pipeline.logger")
    def test_predict_end2end(self, mock_log):
        with patch("sys.stdout", new=StringIO()):
            run_predict_pipeline(self.test_test_piplein_params)

            self.assertTrue(os.path.exists(self.test_test_piplein_params.output_data_path))


# -*- coding: utf-8 -*-
import numpy as np
import os
import time
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.transforms as trn
import torchvision.datasets as dset
import torch.nn.functional as F
import json
from attack_methods import pgd
from models.wrn import WideResNet
from option import BaseOptions

class PrivateOptions(BaseOptions):
    def initialize(self):
        BaseOptions.initialize(self)

        # WRN Architecture
        self.parser.add_argument('--layers', default=28, type=int, help='total number of layers')
        self.parser.add_argument('--widen-factor', default=10, type=int, help='widen factor')
        self.parser.add_argument('--droprate', default=0.0, type=float, help='dropout probability')

# /////////////// Training ///////////////

def train():
    net.train()  # enter train mode
    loss_avg = 0.0
    for bx, by in train_loader:
        bx, by = bx.cuda(), by.cuda()

        adv_bx = adversary_train(net, bx, by)

        # forward
        logits = net(adv_bx)

        # backward
        # scheduler.step()
        optimizer.zero_grad()
        loss = F.cross_entropy(logits, by)
        loss.backward()
        optimizer.step()

        # exponential moving average
        loss_avg = loss_avg * 0.8 + float(loss) * 0.2

    state['train_loss'] = loss_avg

# test function
def test():
    net.eval()
    loss_avg = 0.0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.cuda(), target.cuda()

            adv_data = adversary_test(net, data, target)

            # forward
            output = net(adv_data)
            loss = F.cross_entropy(output, target)

            # accuracy
            pred = output.data.max(1)[1]
            correct += pred.eq(target.data).sum().item()

            # test loss average
            loss_avg += float(loss.data)

    state['test_loss'] = loss_avg / len(test_loader)
    state['test_accuracy'] = correct / len(test_loader.dataset)

# overall_test function
def test_in_testset():
    net.eval()
    loss_avg = 0.0
    correct = 0
    adv_loss_avg = 0.0
    adv_correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.cuda(), target.cuda()

            adv_data = adversary_test(net, data, target)

            # forward
            output = net(data)
            loss = F.cross_entropy(output, target)

            # accuracy
            pred = output.data.max(1)[1]
            correct += pred.eq(target.data).sum().item()

            # test loss average
            loss_avg += float(loss.data)

            # forward
            adv_output = net(adv_data)
            adv_loss = F.cross_entropy(adv_output, target)

            # accuracy
            adv_pred = adv_output.data.max(1)[1]
            adv_correct += adv_pred.eq(target.data).sum().item()

            # test loss average
            adv_loss_avg += float(adv_loss.data)

    state['test_loss'] = loss_avg / len(test_loader)
    state['test_accuracy'] = correct / len(test_loader.dataset)
    state['adv_test_loss'] = adv_loss_avg / len(test_loader)
    state['adv_test_accuracy'] = adv_correct / len(test_loader.dataset)

def test_in_trainset():
    train_loader = torch.utils.data.DataLoader(
    train_data, batch_size=opt.test_bs, shuffle=False,
    num_workers=opt.prefetch, pin_memory=torch.cuda.is_available())
    net.eval()
    loss_avg = 0.0
    correct = 0
    adv_loss_avg = 0.0
    adv_correct = 0
    with torch.no_grad():
        for data, target in train_loader:
            data, target = data.cuda(), target.cuda()

            adv_data = adversary_test(net, data, target)

            # forward
            output = net(data)
            loss = F.cross_entropy(output, target)

            # accuracy
            pred = output.data.max(1)[1]
            correct += pred.eq(target.data).sum().item()

            # test loss average
            loss_avg += float(loss.data)

            # forward
            adv_output = net(adv_data)
            adv_loss = F.cross_entropy(adv_output, target)

            # accuracy
            adv_pred = adv_output.data.max(1)[1]
            adv_correct += adv_pred.eq(target.data).sum().item()

            # test loss average
            adv_loss_avg += float(adv_loss.data)

    state['train_loss'] = loss_avg / len(train_loader)
    state['train_accuracy'] = correct / len(train_loader.dataset)
    state['adv_train_loss'] = adv_loss_avg / len(train_loader)
    state['adv_train_accuracy'] = adv_correct / len(train_loader.dataset)

opt = PrivateOptions().parse()

state = {k: v for k, v in opt._get_kwargs()}

torch.manual_seed(opt.random_seed)
np.random.seed(opt.random_seed)
cudnn.benchmark = True

# # mean and standard deviation of channels of CIFAR-10 images
# mean = [x / 255 for x in [125.3, 123.0, 113.9]]
# std = [x / 255 for x in [63.0, 62.1, 66.7]]

train_transform = trn.Compose([trn.RandomHorizontalFlip(), trn.RandomCrop(32, padding=4),
                               trn.ToTensor(), trn.Normalize(
                                         mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])
test_transform = trn.Compose([trn.ToTensor(), trn.Normalize(
                                         mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])

if opt.dataset == 'cifar10':
    train_data = dset.CIFAR10(opt.dataroot, train=True, transform=train_transform, download=True)
    test_data = dset.CIFAR10(opt.dataroot, train=False, transform=test_transform)
    num_classes = 10
else:
    train_data = dset.CIFAR100(opt.dataroot, train=True, transform=train_transform, download=True)
    test_data = dset.CIFAR100(opt.dataroot, train=False, transform=test_transform)
    num_classes = 100

train_loader = torch.utils.data.DataLoader(
    train_data, batch_size=opt.batch_size, shuffle=True,
    num_workers=opt.prefetch, pin_memory=torch.cuda.is_available())
test_loader = torch.utils.data.DataLoader(
    test_data, batch_size=opt.test_bs, shuffle=False,
    num_workers=opt.prefetch, pin_memory=torch.cuda.is_available())

# Create model
if opt.model == 'wrn':
    net = WideResNet(opt.layers, num_classes, opt.widen_factor, dropRate=opt.droprate)
else:
    assert False, opt.model + ' is not supported.'

start_epoch = opt.start_epoch

if opt.ngpu > 0:
    net = torch.nn.DataParallel(net, device_ids=list(range(opt.ngpu)))
    net.cuda()
    torch.cuda.manual_seed(opt.random_seed)

# Restore model if desired
if opt.load != '':
    if opt.test and os.path.isfile(opt.load):
        net.load_state_dict(torch.load(opt.load))
        print('Appointed Model Restored!')
    else:
        model_name = os.path.join(opt.load, opt.dataset + opt.model +
                                  '_epoch_' + str(start_epoch) + '.pt')
        if os.path.isfile(model_name):
            net.load_state_dict(torch.load(model_name))
            print('Model restored! Epoch:', start_epoch)
        else:
            raise Exception("Could not resume")

epoch_step = json.loads(opt.epoch_step)
lr = state['learning_rate']
optimizer = torch.optim.SGD(
    net.parameters(), lr, momentum=state['momentum'],
    weight_decay=state['decay'], nesterov=True)

# def cosine_annealing(step, total_steps, lr_max, lr_min):
#     return lr_min + (lr_max - lr_min) * 0.5 * (
#             1 + np.cos(step / total_steps * np.pi))
#
#
# scheduler = torch.optim.lr_scheduler.LambdaLR(
#     optimizer,
#     lr_lambda=lambda step: cosine_annealing(
#         step,
#         opt.epochs * len(train_loader),
#         1,  # since lr_lambda computes multiplicative factor
#         1e-6 / opt.learning_rate))  # originally 1e-6

adversary_train = pgd.PGD(epsilon=opt.epsilon * 2, num_steps=opt.num_steps, step_size=opt.step_size * 2).cuda()
adversary_test = pgd.PGD(epsilon=opt.epsilon * 2, num_steps=opt.test_num_steps, step_size=opt.test_step_size * 2).cuda()

if opt.test:
    test_in_testset()
    # test_in_trainset()
    print(state)
    exit()

# Make save directory
if not os.path.exists(opt.save):
    os.makedirs(opt.save)
if not os.path.isdir(opt.save):
    raise Exception('%s is not a dir' % opt.save)

with open(os.path.join(opt.save, "log_" + opt.dataset + opt.model +
                                  '_training_results.csv'), 'w') as f:
    f.write('epoch,time(s),train_loss,test_loss,test_accuracy(%)\n')

print('Beginning Training\n')

# Main loop
best_test_accuracy = 0
for epoch in range(start_epoch, opt.epochs + 1):
    state['epoch'] = epoch

    begin_epoch = time.time()

    train()
    test()

    # Save model
    if epoch > 10 and epoch % 10 == 0:
        torch.save(net.state_dict(),
                   os.path.join(opt.save, opt.dataset + opt.model +
                                '_epoch_' + str(epoch) + '.pt'))

    if state['test_accuracy'] > best_test_accuracy:
        best_test_accuracy = state['test_accuracy']
        torch.save(net.state_dict(),
                   os.path.join(opt.save, opt.dataset + opt.model +
                                '_epoch_best.pt'))

    # Show results
    with open(os.path.join(opt.save, "log_" + opt.dataset + opt.model +
                                      '_training_results.csv'), 'a') as f:
        f.write('%03d,%0.6f,%05d,%0.3f,%0.3f,%0.2f\n' % (
            (epoch),
            lr,
            time.time() - begin_epoch,
            state['train_loss'],
            state['test_loss'],
            100. * state['test_accuracy'],
        ))

    print('Epoch {0:3d} | LR {1:.6f} | Time {2:5d} | Train Loss {3:.3f} | Test Loss {4:.3f} | Test Acc {5:.2f}'.format(
        (epoch),
        lr,
        int(time.time() - begin_epoch),
        state['train_loss'],
        state['test_loss'],
        100. * state['test_accuracy'])
    )

    # Adjust learning rate
    if epoch in epoch_step:
        lr = optimizer.param_groups[0]['lr'] * opt.lr_decay_ratio
        optimizer = torch.optim.SGD(
            net.parameters(), lr, momentum=state['momentum'],
            weight_decay=state['decay'], nesterov=True)
        print("new lr:", lr)


import time
from collections import OrderedDict
from collections.abc import Callable, Sequence
from typing import Any, NamedTuple

DB = "timeseries"
TABLE_RAW = "paii_raw"
TIME_FIELD = "paii_time"

class Field(NamedTuple):
    json_key: str
    store_flag: bool
    db_name: str
    data_type: str
    convert: Callable[[dict[str, Any], str], Any] | None = None

# conversions
def fahrenheit2celsius(data: dict[str, Any], key: str) -> float:
    f = data[key]
    return round((f - 32) * 5 / 9, 2)

def get_response_date(data: dict[str, Any], key: str) -> int:
    return data.get(key, time.time())

def missing_int(data: dict[str, Any], key: str) -> int:
    return data.get(key, -1)

fields = [
    # inserted by the sceduler
    Field(TIME_FIELD, True, TIME_FIELD, "TIMESTAMPTZ NOT NULL"),
    # from the device
    Field("SensorId", False, "SensorId", "VARCHAR"),  # eg "84:f3:eb:7b:c8:ee"
    Field("DateTime", False, "DateTime", "VARCHAR"),  # eg "2020/08/23T10:44:39z"
    Field("Geo", True, "geo", "VARCHAR"),  # eg "PurpleAir-c8ee"
    Field("Mem", False, "Mem", "VARCHAR"),  # eg 18936
    Field("memfrag", False, "memfrag", "VARCHAR"),  # eg 8
    Field("memfb", False, "memfb", "VARCHAR"),  # eg 17568
    Field("memcs", False, "memcs", "VARCHAR"),  # eg 896
    Field("Id", False, "Id", "VARCHAR"),  # eg 4177
    Field("lat", True, "lat", "DOUBLE PRECISION"),  # eg -37.8484
    Field("lon", True, "lon", "DOUBLE PRECISION"),  # eg 145.177399
    Field("Adc", True, "adc", "DOUBLE PRECISION"),  # eg 0.05
    Field("loggingrate", False, "loggingrate", "VARCHAR"),  # eg 15
    Field("place", True, "place", "VARCHAR"),  # eg "outside"
    Field("version", False, "version", "VARCHAR"),  # eg "6.01"
    Field("uptime", False, "uptime", "VARCHAR"),  # eg 242296
    Field("rssi", False, "rssi", "VARCHAR"),  # eg -59
    Field("period", True, "period", "INTEGER"),  # eg 120
    Field("httpsuccess", False, "httpsuccess", "VARCHAR"),  # eg 12169
    Field("httpsends", False, "httpsends", "VARCHAR"),  # eg 12182
    Field("hardwareversion", True, "hardwareversion", "VARCHAR"),  # eg "2.0"
    Field(
        "hardwarediscovered",
        False,
        "hardwarediscovered",
        "VARCHAR",
    ),  # eg "2.0+BME280+PMSX003-B+PMSX003-A"
    Field(
        "current_temp_f",
        True,
        "current_temp_c",
        "DOUBLE PRECISION",
        fahrenheit2celsius,
    ),  # eg 52
    Field("current_humidity", True, "current_humidity", "DOUBLE PRECISION"),  # eg 55
    Field(
        "current_dewpoint_f",
        True,
        "current_dewpoint_c",
        "DOUBLE PRECISION",
        fahrenheit2celsius,
    ),  # eg 36
    Field("pressure", True, "pressure", "DOUBLE PRECISION"),  # eg 1005.28
    Field("p25aqic_b", False, "p25aqic_b", "VARCHAR"),  # eg "rgb(0,228,0)"
    Field("pm2.5_aqi_b", False, "pm2.5_aqi_b", "VARCHAR"),  # eg_aqi_b": 5
    Field("pm1_0_cf_1_b", True, "pm1_0_cf_1_b", "DOUBLE PRECISION"),  # eg 0.39
    Field("p_0_3_um_b", False, "p_0_3_um_b", "VARCHAR"),  # eg 261.79
    Field("pm2_5_cf_1_b", True, "pm2_5_cf_1_b", "DOUBLE PRECISION"),  # eg 1.3 **** µg/m3
    Field("p_0_5_um_b", False, "p_0_5_um_b", "VARCHAR"),  # eg 72.35
    Field("pm10_0_cf_1_b", True, "pm10_0_cf_1_b", "DOUBLE PRECISION"),  # eg 1.72
    Field("p_1_0_um_b", False, "p_1_0_um_b", "VARCHAR"),  # eg 13.05
    Field("pm1_0_atm_b", False, "pm1_0_atm_b", "VARCHAR"),  # eg 0.39
    Field("p_2_5_um_b", False, "p_2_5_um_b", "VARCHAR"),  # eg 2.42
    Field("pm2_5_atm_b", False, "pm2_5_atm_b", "VARCHAR"),  # eg 1.3
    Field("p_5_0_um_b", False, "p_5_0_um_b", "VARCHAR"),  # eg 0.7
    Field("pm10_0_atm_b", False, "pm10_0_atm_b", "VARCHAR"),  # eg 1.72
    Field("p_10_0_um_b", False, "p_10_0_um_b", "VARCHAR"),  # eg 0.0
    Field("p25aqic", False, "p25aqic", "VARCHAR"),  # eg "rgb(0,228,0)"
    Field("pm2.5_aqi", False, "pm2.5_aqi", "VARCHAR"),  # eg_aqi": 1
    Field("pm1_0_cf_1", True, "pm1_0_cf_1", "DOUBLE PRECISION"),  # eg 0.14
    Field("p_0_3_um", False, "p_0_3_um", "VARCHAR"),  # eg 163.63
    Field("pm2_5_cf_1", True, "pm2_5_cf_1", "DOUBLE PRECISION"),  # eg 0.33  **** µg/m3
    Field("p_0_5_um", False, "p_0_5_um", "VARCHAR"),  # eg 45.77
    Field("pm10_0_cf_1", True, "pm10_0_cf_1", "DOUBLE PRECISION"),  # eg 0.42
    Field("p_1_0_um", False, "p_1_0_um", "VARCHAR"),  # eg 7.79
    Field("pm1_0_atm", False, "pm1_0_atm", "VARCHAR"),  # eg 0.14
    Field("p_2_5_um", False, "p_2_5_um", "VARCHAR"),  # eg 0.56
    Field("pm2_5_atm", False, "pm2_5_atm", "VARCHAR"),  # eg 0.33
    Field("p_5_0_um", False, "p_5_0_um", "VARCHAR"),  # eg 0.18
    Field("pm10_0_atm", False, "pm10_0_atm", "VARCHAR"),  # eg 0.42
    Field("p_10_0_um", False, "p_10_0_um", "VARCHAR"),  # eg 0.0
    Field("pa_latency", False, "pa_latency", "VARCHAR"),  # eg 631
    Field("response", False, "response", "VARCHAR"),  # eg 201
    Field(
        "response_date",
        True,
        "response_date",
        "INTEGER",
        get_response_date,
    ),  # eg 1598179477
    Field("latency", True, "latency", "INTEGER", missing_int),  # eg 1459
    Field("key1_response", False, "key1_response", "VARCHAR"),  # eg 200
    Field("key1_response_date", False, "key1_response_date", "VARCHAR"),  # eg 1598179467
    Field("key1_count", False, "key1_count", "VARCHAR"),  # eg 79205
    Field("ts_latency", False, "ts_latency", "VARCHAR"),  # eg 1198
    Field("key2_response", False, "key2_response", "VARCHAR"),  # eg 200
    Field("key2_response_date", False, "key2_response_date", "VARCHAR"),  # eg 1598179470
    Field("key2_count", False, "key2_count", "VARCHAR"),  # eg 79212
    Field("ts_s_latency", False, "ts_s_latency", "VARCHAR"),  # eg 1141
    Field("key1_response_b", False, "key1_response_b", "VARCHAR"),  # eg 200
    Field(
        "key1_response_date_b",
        False,
        "key1_response_date_b",
        "VARCHAR",
    ),  # eg 1598179472
    Field("key1_count_b", False, "key1_count_b", "VARCHAR"),  # eg 79213
    Field("ts_latency_b", False, "ts_latency_b", "VARCHAR"),  # eg 1133
    Field("key2_response_b", False, "key2_response_b", "VARCHAR"),  # eg 200
    Field(
        "key2_response_date_b",
        False,
        "key2_response_date_b",
        "VARCHAR",
    ),  # eg 1598179474
    Field("key2_count_b", False, "key2_count_b", "VARCHAR"),  # eg 79217
    Field("ts_s_latency_b", False, "ts_s_latency_b", "VARCHAR"),  # eg 1136
    Field("wlstate", False, "wlstate", "VARCHAR"),  # eg "Connected"
    Field("status_0", True, "status_0", "INTEGER"),  # eg 2
    Field("status_1", True, "status_1", "INTEGER"),  # eg 2
    Field("status_2", True, "status_2", "INTEGER"),  # eg 2
    Field("status_3", True, "status_3", "INTEGER"),  # eg 2
    Field("status_4", True, "status_4", "INTEGER"),  # eg 2
    Field("status_5", True, "status_5", "INTEGER"),  # eg 2
    Field("status_6", True, "status_6", "INTEGER", missing_int),  # eg 2
    Field("status_7", True, "status_7", "INTEGER"),  # eg 0
    Field("status_8", True, "status_8", "INTEGER"),  # eg 2
    Field("status_9", True, "status_9", "INTEGER"),  # eg 2
    Field("ssid", False, "ssid", "VARCHAR"),  # eg "apocalypse
]

def gen_stored(fields: list[Field] = fields):
    yield from (f for f in fields if f.store_flag)

def compose_create(
    table_name: str,
    time_field: str,
    fields: list[Field] = fields,
) -> str:
    fdesc = ",\n".join([f"{f.db_name} {f.data_type}" for f in gen_stored()])
    sql = f"""CREATE TABLE IF NOT EXISTS {table_name} (
{fdesc},
PRIMARY KEY({time_field})
);
    """
    return sql

def compose_insert(field_names: Sequence, table_name: str) -> str:
    """compose parameterized insert SQL

    Args:
        field_names (Sequence): database table field names
        table_name (str): database table name.

    Returns:
        str: insert SQL.
    """
    fields = ", ".join(field_names)
    placeholders = ", ".join([f"${i+1}" for i in range(len(field_names))])
    sql = f"INSERT INTO {table_name} ({fields}) values ({placeholders})"
    return sql

def convert_data(data: dict[str, Any], fields: list[Field] = fields) -> OrderedDict:
    """return filtered and ordered device data

    Args:
        data (Dict[str, Any]): raw dictionary directly from device.
        fields (List[Field], optional): fields specification

    Returns:
        OrderedDict[str, Any]: {db_key: converted_value, ...} items() will return
        in the same order as SQL commands assuming they are all based on the same
        field list.
    """

    missing = []

    def convert(data: dict[str, Any], field: Field) -> Any:
        if field.convert:
            # custom converstion function, takes care of missing data
            return field.convert(data, field.json_key)
        if field.json_key in data:
            return data[field.json_key]
        else:
            missing.append(field.json_key)
            return None

    missing.clear()
    result = OrderedDict({f.db_name: convert(data, f) for f in gen_stored()})
    # if missing:
    #     print(f"fields were missing: {missing}")
    return result


import numpy as np
import matplotlib
from matplotlib.colors import ListedColormap

SLACred = '#8C1515'
SLACgrey = '#53565A'
SLACblue = '#007C92'
SLACteal = '#279989'
SLACgreen = '#8BC751'
SLACyellow = '#FEDD5C'
SLACorange = '#E04F39'
SLACpurple = '#53284F'
SLAClavender = '#765E99'
SLACbrown = '#5F574F'

SLACcolors = [SLACred,
              SLACblue,
              SLACteal,
              SLACgreen,
              SLACyellow,
              SLACgrey,
              SLACorange,
              SLACpurple,
              SLAClavender,
              SLACbrown,
]

# SLACsage = [199./256, 209./256, 197./256]
white = [256./256, 256./256, 256./256]
SLACpaloverde = [39./256, 153./256, 137./256]

matplotlib.cm.register_cmap('SLACverde',
                            ListedColormap(np.array([np.interp(np.linspace(0, 1, 256),
                                                               [0, 1],
                                                               [whiteV, pvV])
                                                     for whiteV, pvV in zip(white, SLACpaloverde)]).T,
                                           name = 'SLACverde'))

LaTeXflavor = {"numu": r'$\nu_\mu$',
               "numubar": r'$\bar{\nu}_\mu$',
               "nue": r'$\nu_e$',
               "nuebar": r'$\bar{\nu}_e$',
               "nutau": r'$\nu_\tau$',
               "nutaubar": r'$\bar{\nu}_\tau$'}

matplotlib.rc('axes', **{"prop_cycle": matplotlib.cycler(color = SLACcolors)})
matplotlib.rc('image', **{"cmap": 'SLACverde'})
matplotlib.rc('font', **{"family": 'sans-serif',
                         "sans-serif": 'Arial',
                         "size": 16,
                         "weight": 'bold'})
matplotlib.rc('text', **{"usetex": True})


from pathlib import Path

from vesper.tests.test_case import TestCase
from vesper.util.preference_manager import PreferenceManager
import vesper.tests.test_utils as test_utils

_DATA_DIR_PATH = Path(test_utils.get_test_data_dir_path(__file__))
_PREFERENCE_FILE_PATH = _DATA_DIR_PATH / 'Preferences.yaml'
_EMPTY_PREFERENCE_FILE_PATH = _DATA_DIR_PATH / 'Empty Preferences.yaml'
_NON_MAPPING_PREFERENCE_FILE_PATH = \
    _DATA_DIR_PATH / 'Non Mapping Preferences.yaml'
_MALFORMED_PREFERENCE_FILE_PATH = _DATA_DIR_PATH / 'Malformed Preferences.yaml'

class PreferenceManagerTests(TestCase):

    def test_get(self):

        preferences = _get_preferences(_PREFERENCE_FILE_PATH)

        cases = (
            ('one', 1),
            ('category_a.two', 2),
            ('category_a.three', 'three'),
            ('category_a.category_b.forty_five', 45),
            ('category_a.category_b.fifty six', 56),
            ('category_a.category_b', {'forty_five': 45, 'fifty six': 56})
        )

        for p in preferences:
            for name, value in cases:
                self.assertTrue(name in p)
                self.assertEqual(p[name], value)
                self.assertEqual(p.get(name), value)

    def test_get_of_nonexistent_preferences(self):

        preferences = _get_preferences(_PREFERENCE_FILE_PATH)

        cases = (
            'bobo',
            'category_a.bobo'
        )

        for p in preferences:
            for name in cases:
                self.assertFalse(name in p)
                self.assertRaises(KeyError, p.__getitem__, name)
                self.assertIsNone(p.get(name))
                self.assertEqual(p.get(name, 10), 10)

    def test_empty_preference_file(self):
        self._test_bad_preference_file(_EMPTY_PREFERENCE_FILE_PATH)

    def _test_bad_preference_file(self, file_path):
        preferences = _get_preferences(file_path)
        for p in preferences:
            self.assertEqual(len(p), 0)

    def test_malformed_preference_file(self):
        self._test_bad_preference_file(_MALFORMED_PREFERENCE_FILE_PATH)

    def test_non_mapping_preference_file(self):
        self._test_bad_preference_file(_NON_MAPPING_PREFERENCE_FILE_PATH)

    def test_nonexistent_preference_file(self):
        manager = PreferenceManager()
        manager.load_preferences_from_file('nonexistent')
        self.assertEqual(len(manager.preferences), 0)

def _get_preferences(file_path):

    # Create preferences from file.
    manager = PreferenceManager.create_for_file(file_path)
    preferences_a = manager.preferences

    # Create preferences from YAML.
    with open(file_path) as file_:
        yaml = file_.read()
    manager = PreferenceManager.create_for_yaml(yaml)
    preferences_b = manager.preferences

    return (preferences_a, preferences_b)


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.ui import WebDriverWait
import random

def get_sore_and_Price(store_id,internet_id):
    driver = webdriver.Chrome('C:/Users/cunzh/Desktop/chromedriver.exe') ## you should change the path before you run
    start_page = driver.get("https://www.homedepot.com/l/")
    driver.find_element_by_id("storeSearchBox").send_keys(store_id)
    driver.find_element_by_class_name("sfSearchbox__button").click()
    time.sleep(random.randint(3,5))
    Message=''
    try:
        store = driver.find_element_by_class_name('sfstores')
        store_name = store.find_element_by_class_name('sfstorename').text
    #print(store.get_attribute("outerHTML"))
    except:
        price="NA"
        Message="store cannot be found"
    else:

        a = store.find_element_by_class_name('sfstorelinks').find_element_by_tag_name('a')
        time.sleep(random.randint(3,5))   #time.sleep are pretening human behavior; human spend different time on different web. So this website will not recognize this as a bot.
        a.click()                         #Randint gives us random integer 3 to 5
        time.sleep(random.randint(3,5))

        driver.find_element_by_id("headerSearch").send_keys(internet_id)
        time.sleep(random.randint(3,5))
        driver.find_element_by_id("headerSearchButton").click()
        time.sleep(random.randint(3,5))
        try:
            content = driver.find_element_by_class_name("price-detailed__wrapper")
            # print(content.get_attribute('innerHTML'))
            spans = content.find_elements_by_tag_name('span')
            if len(spans) != 3:
                price='NA'
                Message='price cannot be found'
            else:
                a = spans[1]
                b = spans[2]
                price = a.text + '.' + b.text

        except:
            price='NA'
            Message='price cannot be found'

    return store_id,price,Message

# In[ ]:

# We can test the code by using follwing example:
store_list = ['954', '907', '6917']
test_list = ['302895490', '302895488', '100561401', '206809290']
list1=[]
for store in store_list:
        for item in test_list:
            list1.append(get_sore_and_Price(store,item))

# In[ ]:

list1


from waterworld.waterworld import env as custom_waterworld
from potential_field.potential_field_policy import PotentialFieldPolicy
from utils import get_frames

from pettingzoo.utils import average_total_reward

from multiprocessing import Pool, cpu_count

import tqdm
import numpy as np
from matplotlib import pyplot as plt
import json

n_coop_options = [1, 2]
n_sensor_options = [1, 2, 5, 20, 30]
angle_options = [("randomize_angle",False),("randomize_angle",True),
                ("spin_angle",0),("spin_angle",0.1),("spin_angle",0.5),("spin_angle",1)]
obs_weighting_options=[1, 0.5]
poison_weighting_options=[1, 0.5]
barrier_weighting_options=[1, 0.5]
food_weighting_options=[1, 0.5]

def test_policy(config, rounds=100):
    env = custom_waterworld(**config["env_config"])
    policy = PotentialFieldPolicy(**config["potential_field_config"]).get_movement_vector
    for i in tqdm.tqdm(range(rounds)):
        reward_sum, frame_list = get_frames(env, policy)
        config["rewards"].append(reward_sum)
    env.close()

    with open(f"potential_field/test_main/{config['config_index']}.json", "x") as f:
        json.dump(config, f, indent=4)

def get_configs():
    configs = []
    i=0
    for n_coop in n_coop_options:
        for n_sensor in n_sensor_options:
            for angle_config in angle_options:
                configs.append({"env_config":
                            {"n_coop": n_coop,"n_sensors": n_sensor,},
                            "potential_field_config":{
                                "n_sensors": n_sensor,
                                angle_config[0]: angle_config[1],
                            },
                                "rewards": [],
                                "config_index": i
                            })
                i += 1
        for obs_weight in obs_weighting_options:
            for poison_weight in poison_weighting_options:
                for barrier_weight in barrier_weighting_options:
                    for food_weight in food_weighting_options:
                        configs.append({"env_config":
                                    {"n_coop": n_coop,"n_sensors": 30,},
                                    "potential_field_config":{
                                        "n_sensors": 30,
                                        "obs_weight": obs_weight,
                                        "poison_weight": poison_weight,
                                        "barrier_weight": barrier_weight,
                                        "food_weight": food_weight
                                    },
                                        "rewards": [],
                                        "config_index": i
                                    })
                        i += 1
    return configs

def get_main_configs():
    configs = []
    i=0
    for n_coop in n_coop_options:
        for n_sensor in n_sensor_options:
            for angle_config in angle_options:
                configs.append({"env_config":
                            {"n_coop": n_coop,"n_sensors": n_sensor,},
                            "potential_field_config":{
                                "n_sensors": n_sensor,
                                angle_config[0]: angle_config[1],
                            },
                                "rewards": [],
                                "config_index": i
                            })
                i += 1

    return configs

def get_env_configs():
    configs = []
    i=0
    for n_coop in n_coop_options:
        for n_sensor in n_sensor_options:
                configs.append({"env_config":
                            {"n_coop": n_coop,"n_sensors": n_sensor,},
                                "rewards": [],
                                "config_index": i
                            })
                i += 1

    return configs

def test_random_env(config, rounds=100):
    env = custom_waterworld(**config["env_config"])
    action_space = env.action_space("pursuer_0")
    def policy(obs):
        return action_space.sample()

    for i in tqdm.tqdm(range(rounds)):
        reward_sum, frame_list = get_frames(env, policy)
        config["rewards"].append(reward_sum)
    env.close()

    with open(f"potential_field/test_random/{config['config_index']}.json", "x") as f:
        json.dump(config, f, indent=4)

if __name__ == "__main__":
    configs = get_env_configs()

    with Pool(processes=int(cpu_count() - 2)) as pool:
        for _ in tqdm.tqdm(pool.imap_unordered(test_random_env, configs), total=len(configs)):
            pass


from scipy.sparse import csr_matrix

from .text import WordPieceParser

from collections.abc import Mapping, Iterable

class RecordVectorMap(Mapping):

    def __init__(self, records, wp_model_path, vec_format='bag-of-words'):

        text_parser = WordPieceParser(wp_model_path)

        self.rec_seq_map, self.record_vecs = self.rec2vecs(records, text_parser, vec_format)

    def rec2vecs(self, records, text_parser, vec_format):

        rec_seq_map = {}
        cols, rows, data = [], [], []

        col_dim = 0 if vec_format=='sequence' else text_parser.vocab_size

        for rec_seq, (rec_id, rec_text) in enumerate(records):

            rec_seq_map[rec_id] = rec_seq

            parsed = text_parser.parse(rec_text, parse_format=vec_format)

            if vec_format=='sequence':

                if len(parsed)!=0:
                    rows.extend([rec_seq]*len(parsed))
                    cols.extend(list(range(len(parsed))))
                    data.extend(parsed)
                    if len(parsed)>col_dim:
                        col_dim = len(parsed)

            else:

                for wp_id, tf in parsed.items():
                    rows.append(rec_seq)
                    cols.append(wp_id)
                    data.append(tf)

        record_vecs = csr_matrix((data, (rows, cols)), shape=(len(records), col_dim))

        return rec_seq_map, record_vecs

    def __getitem__(self, key):
        if isinstance(key, str):
            return self.get_by_seqs(self.rec_seq_map[key])
        elif isinstance(key, Iterable):
            return self.get_by_seqs([self.rec_seq_map[a_key] for a_key in key])
        else:
            raise TypeError('Key must be string (key of record) or iterable (list of key of record).')

    def get_by_seqs(self, key):
        if isinstance(key, int):
            return self.record_vecs[key]
        elif isinstance(key, Iterable):
            return self.record_vecs[key]
        else:
            raise TypeError('Seqs must be int (seq of record) or iterable (list of seq of record).')

    def __iter__(self):
        return iter(self.record_vecs)

    def __len__(self):
        return len(self.rec_seq_map)


from typing import Any
from random import randint, random
import pygame as py

from engine.game_engine import Pygame
from engine.color import Color
from engine.game_objects import IGameObject
from engine.game_objects.modules import IAnimationModule, ICollisionModule
from engine.image import SpriteSheet
from engine.ui.text import TypeWriterText
from engine.map import LoopingMap

from src.player.player import Player

class UI_NPC_Text(TypeWriterText):
    def __init__(self, game: Pygame = None):
        super().__init__("We need to push forward! We need to push forward!", (200, 200, 200), game.fonts["8bit"], (135, 590), 10, game=game)

class UI_NPC_Text_Box(IGameObject):
    def __init__(self, game: Pygame = None):
        super().__init__("ui", game)
        self.pos = (120, 577)
        self.add_layer(f"{self.game.game_dir}\\data\\images\\ui\\ui_npc_text_box.png")
        self.rect = self.get_layer_image(0).get_rect(topleft=self.pos)

class UI_NPC_Box(IGameObject):
    def __init__(self, game: Pygame = None):
        super().__init__("ui", game)
        self.pos = (10, 665)
        # self.add_layer(f"{self.game.game_dir}/data/images/npc_board.png")
        self.add_layer(py.Surface((100, 100), flags=py.SRCALPHA))
        self.add_layer(py.Surface((100, 100), flags=py.SRCALPHA))
        self.add_layer(py.Surface((100, 100), flags=py.SRCALPHA))
        # self.add_layer(f"{self.game.game_dir}/data/images/npc_box.png")
        self.npc = ""
        self.npcs = {
            "red": py.image.load(f"{self.game.game_dir}/data/images/ui/npcs/red_npc.png")
        }
        self.anim_atlas = SpriteSheet(f"{self.game.game_dir}/data/images/ui/noise_atlas.png", 100, 100).get_image_array()
        self.rect = self.get_layer_image(1).get_rect(bottomleft=self.pos)

        self.set_module("animation", IAnimationModule(self, False))
        self.get_module("animation").add_animation_by_dict(
            "noise",
            {
                "layer": 1,
                "frames": [self.anim_atlas[0], self.anim_atlas[1], self.anim_atlas[2], self.anim_atlas[3],
                           self.anim_atlas[4], self.anim_atlas[5], self.anim_atlas[6], self.anim_atlas[7]],
                "frame_time": 40,
                "loop": False,
                "callback": self.noise_anim_ended
            }
        )

        self.get_module("animation").add_animation_by_dict(
            "noise_reversed",
            {
                "layer": 1,
                "frames": [self.anim_atlas[7], self.anim_atlas[6], self.anim_atlas[5], self.anim_atlas[4],
                           self.anim_atlas[3], self.anim_atlas[2], self.anim_atlas[1], self.anim_atlas[1]],
                "frame_time": 40,
                "loop": False,
                "callback": self.noise__reversed_anim_ended
            }
        )
        self.text = UI_NPC_Text(self.game)
        self.border = UI_NPC_Text_Box(self.game)
        self.game.objects.add("ui_npc_text", self.text)
        self.game.objects.add("ui_npc_text_box", self.border)
        self.get_module("animation").should_animate = True
        self.get_module("animation").play("noise")

    def noise_anim_ended(self):
        self.npc = "red"
        if self.npc != "":
            self.set_layer(self.npcs[self.npc], layer_id=0)
            self.set_layer(self.anim_atlas[8], layer_id=1)

    def noise__reversed_anim_ended(self):
        self.npc = ""
        self.set_layer(py.Surface((100, 100), py.SRCALPHA), layer_id=0)
        self.set_layer(py.Surface((100, 100), py.SRCALPHA), layer_id=1)

class UI_BG(IGameObject):
    def __init__(self, game=None):
        super().__init__("ui", game)
        self.pos = (0, 675)
        self.add_layer(f"{self.game.game_dir}/data/images/ui_bg.png")
        self.rect = self.get_layer_image(0).get_rect(bottomleft=self.pos)

# Rock Object #
class Rock(IGameObject):
    def __init__(self, game: Pygame):
        super(Rock, self).__init__('env', game)
        self.set_module("collision", ICollisionModule(self, False))
        self.pos = (self.game.screen.get_width(), randint(0, self.game.screen.get_height()))
        self.vel = (randint(-2,-1), randint(-1, 1) * (random() * 0.5))
        spr = py.image.load(f"{self.game.game_dir}/data/images/rock.png").convert_alpha()
        scale = max(25, random() * 48)
        spr = py.transform.scale(spr, (scale, scale))
        spr = py.transform.rotate(spr, randint(-25, 25))
        self.add_layer(spr)
        self.rect = self.get_layer_image(self.primary_layer).get_rect(center=self.pos)
        self.mask = py.mask.from_surface(self.get_layer_image(0))

    def update(self, *args, **kwargs) -> None:
        self.pos = (self.pos[0] + self.vel[0], self.pos[1] + self.vel[1])
        self.rect = self.get_layer_image(0).get_rect(center=self.pos)
        super(Rock, self).update()

class RockSpawner(IGameObject):
    def __init__(self, group: str = "handler", game: Pygame = None):
        super().__init__(group, game)
        self.spawn_timing = 2000
        self.last_spawn_time = self.game.time

    def update(self, *args: Any, **kwargs: Any) -> None:
        if self.last_spawn_time + self.spawn_timing <= self.game.time:
            self.game.objects.add(f"rock_{randint(0, 999999)}", Rock(self.game))
            self.last_spawn_time = self.game.time
        return super().update(*args, **kwargs)

# MINIMAL RUNNING EXAMPLE #
# Main Game Engine Object #
class Game(Pygame):
    def __init__(self):
        super(Game, self).__init__(1200, 675, "Space Game", fps_target=60)
        self.add_group("handler")
        self.add_group("map")
        self.add_group("player")
        self.add_group("player_projectile")
        self.add_group("env")
        self.add_group("enemy")
        self.add_group('ui')
        self.fonts["8bit"] = self.get_font("8-BIT WONDER.ttf", 17)
        # SETUP GAME AXIS CONTROLS (IControlModule().get_axis("move_left"))
        self.axis = {'move_left': {py.K_a: -1, py.K_d: 1}, 'move_up': {py.K_w: -1, py.K_s: 1}}
        py.event.set_grab(True)
        py.mouse.set_pos((100, self.screen.get_height() / 2))
        py.mouse.set_visible(False)
        self.load_data()
        self.start_game_loop()

    def load_data(self):
        super(Game, self).load_data() # Required to set the base dir of the game for easy access in objects without recalculating where to split the path (self.game_dir)
        self.objects['map'] = LoopingMap(self, "data/images/background.png", [50, 0]) # Looping map uses the images size to calculate its looping. You may need to rescale your image to fit your game area (Window scaling will handle it after as long as it covers the initial screen). Supports vertical or horizontal but not both (Hopefully in future revisions)
        self.objects["player"] = Player((0, self.screen.get_height() / 2), self) # Adds a GameObject to the ObjectHandler so that update and draw calls are triggered correctly
        self.objects.add("rock_spawner", RockSpawner(game=self))
        self.objects["ui_bg"] = UI_BG(self)
        self.objects['ui_npc_box'] = UI_NPC_Box(self)
        # self.objects['ui_npc_text'] = UI_NPC_Text(self)

    def draw(self):
        self.screen.fill(Color(1, 1, 1, 1).RGB) # self.screen.fill((255, 255, 255)). Color class is used mostly for storing colors to easily recall but may get more features later
        super(Game, self).draw() # Required to call the draw function for registered objects
        # UNCOMMENT FOR RECT DEBUGGING
        if self.debug:
            for group in self.groups.keys():
                for sprite in self.groups[group]:
                    self.debug.debug_collision(sprite)

        super(Game, self).render_display() # Required to call the render update of the display (py.display.flip())

    def update(self):
        super(Game, self).update()
        for event in py.event.get():
            # COMMENT TO REMOVE MANUAL WINDOW RESIZING SUPPORT
            if event.type == py.VIDEORESIZE:
                self.windowSize = (event.w, event.h)
                py.display._resize_event(event)
            if event.type == py.QUIT:
                self.quit()
            if event.type == py.KEYDOWN:
                # UNCOMMENT TO TOGGLE DEBUGGING
                if event.key == py.K_c: self.debug.set_debug(not self.debug._debug)
                if event.key == py.K_p:
                    npc_ui = self.objects.get("ui_npc_box")
                    if npc_ui.npc == "":
                        npc_ui.get_module("animation").play("noise")
                    else:
                        self.objects["ui_npc_box"].get_module("animation").play("noise_reversed")
                if event.key == py.K_ESCAPE:
                    self.quit()

if __name__ == '__main__':
    g = Game()


import streamlit as st
import calculator_logic

st.title("Calculator App")

num1 = st.number_input("Enter the first number:")
num2 = st.number_input("Enter the second number:")
operation = st.selectbox("Select an operation", calculator_logic.OPERATIONS)

if st.button("Calculate"):
    result = calculator_logic.calculate(num1, num2, operation)
    st.success(f"The result is {result}")

# Define a function to display the signature
def display_signature():
    st.markdown(
        """
        <style>
        .signature {
            font-size: 1rem;
            font-style: italic;
            text-align: center;
            padding: 1rem 0;
            color: #333;
            transition: color 0.5s ease-in-out;
        }
        .signature:hover {
            color: #007bff;
        }
        </style>
        """
        , unsafe_allow_html=True
    )
    st.markdown(
        """
        <div class="signature">
        Made with ❤ by Shib Kumar Saraf
        </div>
        """
        , unsafe_allow_html=True
    )

# Add the signature to your Streamlit app
display_signature()


import sys
from collections import defaultdict

def tpsortutil(u, visited, stack, cur):
    visited[u] = True
    for i in graph[u]:
        if not visited[i]:
            tpsortutil(i, visited, stack, cur)
        elif i in cur:
            return
    stack.append(u)

def topologicalsort(graph, vertices):
    visited = [False] * vertices
    stack = []
    for i in range(vertices):
        cur = set()
        if not visited[i] and graph[i]:
            tpsortutil(i, visited, stack, cur)
    del cur
    stack = stack[::-1]
    print(stack)

if __name__ == "__main__":
    vertices = int(input())
    graph = defaultdict(list)
    edges = int(input())
    for _ in range(edges):
        edge = [int(x) for x in input().split()]
        graph[edge[0]].append(edge[1])
    print(graph)
    topologicalsort(graph, vertices)


"""
URL configuration for backend project.

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.urls import include, path
from drf_yasg import openapi
from drf_yasg.views import get_schema_view
from rest_framework import permissions

schema_view = get_schema_view(
    openapi.Info(
        title="SoC Portal API",
        default_version="v1",
        description="Test description",
        terms_of_service="https://www.google.com/policies/terms/",
        contact=openapi.Contact(email="contact@wncc.local"),
        license=openapi.License(name="BSD License"),
    ),
    permission_classes=[],
    public=True,
)

urlpatterns = [
    path("admin/", admin.site.urls),
    path("api/accounts/", include("accounts.urls")),
    path("api/dashboard/", include("dashboard.urls")),
    path("api/projects/", include("projects.urls")),
]

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
urlpatterns += [
    path(
        "swagger<format>/", schema_view.without_ui(cache_timeout=0), name="schema-json"
    ),
    path(
        "swagger/",
        schema_view.with_ui("swagger", cache_timeout=0),
        name="schema-swagger-ui",
    ),
    path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
]
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import numpy as np
from sklearn.linear_model import LogisticRegression
from random import randrange
from math import ceil, floor

# Загрузка данных из текстового файла
data = np.genfromtxt('данные двумерная модель.txt', skip_header=1)  # Пропустить первую строку с названиями столбцов

# Разделение данных на факторы (x) и зависимую переменную (y)
X = data[:, :-1]  # Первые 6 столбцов
y = data[:, -1]   # Последний столбец

# Создание и обучение модели множественной регрессии
model = LogisticRegression()
model.fit(X, y)

# Вывод коэффициентов регрессии
print("Коэффициенты регрессии:")
print("a1, a2:", [round(x, 3) for x in model.coef_[0]])
print("b (пересечение):", round(model.intercept_[0], 3))

X_min = [float('inf'), float('inf')]
X_max = [float('-inf'), float('-inf')]
for x in X:
    for i in range(len(x)):
        if x[i] < X_min[i]:
            X_min[i] = x[i]
        if x[i] > X_max[i]:
            X_max[i] = x[i]

R_X = []
for i in range(len(X_min)):
    R_X.append(randrange(ceil(X_min[i]), floor(X_max[i])))

print(R_X)

# Спрогнозировать новое значение y на основе заданных факторов (замените значения x_new)
x_new = np.array([R_X])
y_pred = model.predict(x_new)
print(f"Прогнозное значение y для новых данных: {y_pred[0]:.2f}")

print(f'{model.coef_[0][0]:.3f} {model.coef_[0][1]:.3f}, {model.intercept_[0]:.3f}')


import json
import os
import docx

with open(f'disciplinas.json') as f:
    data = json.load(f)

# print(df.columns.values)
for index, discpln in data.items():

    print(f'{discpln["sigla"]} - {discpln["nome"]}')

    doc = docx.Document()
    doc.add_heading(f'{discpln["sigla"]} - {discpln["nome"]}')
    doc.add_heading(f'{discpln["nome_en"]}', level=3)
    doc.add_paragraph()

    # Dados gerais
    p = doc.add_paragraph(style = 'List Bullet')
    p.add_run(f'Créditos-aula: {discpln["CA"]}\n')
    p.add_run(f'Créditos-trabalho: {discpln["CT"]}\n')
    p.add_run(f'Carga horária: {discpln["CH"]}\n')
    p.add_run(f'Ativação: {discpln["ativacao"]}\n')
    p.add_run(f'Departamento: {discpln["departamento"]}\n')

    # Cursos e semestres ideais
    cs = f'Curso (semestre ideal):'
    for curso, semestre in discpln["semestre"].items():
        cs += f' {curso} ({semestre}),'
    p.add_run(cs[:-1])

    # Objetivos
    doc.add_heading(f'Objetivos', level=2)
    doc.add_paragraph(f'{discpln["objetivos"]}')
    if discpln["abstract"]:
        p = doc.add_paragraph()
        p.add_run(f'{discpln["objectives"]}').italic = True

    # Docentes
    doc.add_heading(f'Docente(s) Responsável(eis) ', level=2)
    profs = discpln["docentes"]
    nprofs = discpln["ndoc"]
    if nprofs:
        p = doc.add_paragraph(style='List Bullet')
        for i in range(nprofs-1):
            p.add_run(f'{profs[i]}\n')
        p.add_run(f'{profs[-1]}')

    # programa resumido
    doc.add_heading(f'Programa resumido', level=2)
    doc.add_paragraph(f'{discpln["resumo"]}')
    if discpln["abstract"]:
        p = doc.add_paragraph()
        p.add_run(f'{discpln["abstract"]}').italic = True

    # programa
    doc.add_heading(f'Programa', level=2)
    doc.add_paragraph(f'{discpln["programa"]}')
    if discpln["program"]:
        p = doc.add_paragraph()
        p.add_run(f'{discpln["program"]}').italic = True

    # avaliação
    doc.add_heading('Avaliação', level=2)
    p = doc.add_paragraph( style='List Bullet')
    p.add_run('Método: ').bold = True
    p.add_run(f'{discpln["metodo"]}\n')
    p.add_run('Critério: ').bold = True
    p.add_run(f'{discpln["criterio"]}\n')
    p.add_run('Norma de recuperação: ').bold = True
    p.add_run(f'{discpln["exame"]}')

    # bibliografia
    doc.add_heading('Bibliografia', level=2)
    doc.add_paragraph(f'{discpln["bibliografia"]}')

    # Requisitos
    nr = discpln['requisitos']
    if nr:
        doc.add_heading('Requisitos', level=2)
        p = doc.add_paragraph(style='List Bullet')
        for k, req in nr.items():
            p.add_run(f"{req['sigla']} - {req['nome']} ({req['tipo']})\n")

    # salvando
    try:
        os.mkdir(f'../assets/disciplinas/')
    except  FileExistsError:
        pass
    docname = f'../assets/disciplinas/{discpln["sigla"]}.docx'
    doc.save(docname)

    # exportando pdf
    os.system(f'abiword --to=pdf {docname}')

    # break


from sklearn.metrics import pairwise_distances
import numpy as np
import pandas as pd
from scipy.sparse import spmatrix
from anndata import AnnData
from scipy.stats import rankdata
from typing import Optional
from . import logger
from .symbols import NOVEL, REMAIN, UNASSIGN

class Distance():
    """
    Class that deals with the cross-dataset cell-by-cell-type distance matrix.

    Parameters
    ----------
    dist_mat
        Cell-by-cell-type distance matrix.
    cell
        Cell meta-information including at least `'dataset'`, `'ID'` and `'cell_type'`.
    cell_type
        Cell type meta-information including at least `'dataset'` and `'cell_type'`.

    Attributes
    ----------
    dist_mat
        A cell-by-cell-type distance matrix.
    cell
        Cell meta-information including `'dataset'`, `'ID'` and `'cell_type'`.
    cell_type
        Cell type meta-information including `'dataset'` and `'cell_type'`.
    n_cell
        Number of cells involved.
    n_cell_type
        Number of cell types involved.
    shape
        Tuple of number of cells and cell types.
    assignment
        Assignment of each cell to the most similar cell type in each dataset (obtained through the `assign` method).
    """
    def __init__(self, dist_mat: np.ndarray, cell: pd.DataFrame, cell_type: pd.DataFrame):
        self.dist_mat = dist_mat
        if cell.shape[0] != self.dist_mat.shape[0]:
            raise ValueError(
                    f"🛑 Number of cells in `cell` does not match the cell number in `dist_mat`")
        if cell_type.shape[0] != self.dist_mat.shape[1]:
            raise ValueError(
                    f"🛑 Number of cell types in `cell_type` does not match the cell type number in `dist_mat`")
        if not {'dataset', 'ID', 'cell_type'}.issubset(set(cell.columns)):
            raise KeyError(
                    f"🛑 Please include `'dataset'`, `'ID'` and `'cell_type'` as the cell meta-information")
        if not {'dataset', 'cell_type'}.issubset(set(cell_type.columns)):
            raise KeyError(
                    f"🛑 Please include `'dataset'` and `'cell_type'` as the cell type meta-information")
        self.cell = cell
        self.cell_type = cell_type

    @property
    def n_cell(self) -> int:
        """Number of cells."""
        return self.dist_mat.shape[0]

    @property
    def n_cell_type(self) -> int:
        """Number of cell types."""
        return self.dist_mat.shape[1]

    @property
    def shape(self) -> tuple:
        """Numbers of cells and cell types."""
        return self.dist_mat.shape

    def __repr__(self):
        lend = len(np.unique(self.cell_type.dataset))
        if lend > 1:
            base = f"Cross-dataset distance matrix between {self.n_cell} cells and {self.n_cell_type} cell types from {lend} datasets"
        else:
            base = f"Distance matrix between {self.n_cell} cells and {self.n_cell_type} cell types"
        base += f"\n    dist_mat: distance matrix between {self.n_cell} cells and {self.n_cell_type} cell types"
        base += f"\n    cell: cell meta-information ({str(list(self.cell.columns))[1:-1]})"
        base += f"\n    cell_type: cell type meta-information ({str(list(self.cell_type.columns))[1:-1]})"
        if hasattr(self, 'assignment'):
            base += f"\n    assignment: data frame of cross-dataset cell type assignment"
        return base

    @staticmethod
    def from_adata(adata: AnnData, dataset: str, cell_type: str, use_rep: Optional[str] = None, metric: Optional[str] = None, n_jobs: Optional[int] = None, check_params: bool = True, **kwargs):
        """
        Generate a :class:`~cellhint.distance.Distance` object from the :class:`~anndata.AnnData` given.

        Parameters
        ----------
        adata
            An :class:`~anndata.AnnData` object containing different datasets/batches and cell types.
            In most scenarios, the format of the expression `.X` in the AnnData is flexible (normalized, log-normalized, z-scaled, etc.).
            However, when `use_rep` is specified as `'X'` (or `X_pca` is not detected in `.obsm` and no other latent representations are provided), `.X` should be log-normalized (to a constant total count per cell).
        dataset
            Column name (key) of cell metadata specifying dataset information.
        cell_type
            Column name (key) of cell metadata specifying cell type information.
        use_rep
            Representation used to calculate distances. This can be `'X'` or any representations stored in `.obsm`.
            Default to the PCA coordinates if present (if not, use the expression matrix `X`).
        metric
            Metric to calculate the distance between each cell and each cell type. Can be `'euclidean'`, `'cosine'`, `'manhattan'` or any metrics applicable to :func:`sklearn.metrics.pairwise_distances`.
            Default to `'euclidean'` if latent representations are used for calculating distances, and to `'correlation'` if the expression matrix is used.
        n_jobs
            Number of CPUs used. Default to one CPU. `-1` means all CPUs are used.
        check_params
            Whether to check (or set the default) for `dataset`, `cell_type`, `use_rep` and `metric`.
            (Default: `True`)
        **kwargs
            Other keyword arguments passed to :func:`sklearn.metrics.pairwise_distances`.

        Returns
        ----------
        :class:`~cellhint.distance.Distance`
            A :class:`~cellhint.distance.Distance` object representing the cross-dataset cell-by-cell-type distance matrix.
        """
        #Use `check_params = False` if `dataset`, `cell_type`, `use_rep` and `metric` are already provided correctly.
        if check_params:
            if dataset not in adata.obs:
                raise KeyError(
                        f"🛑 '{dataset}' is not found in the provided AnnData")
            if cell_type not in adata.obs:
                raise KeyError(
                        f"🛑 '{cell_type}' is not found in the provided AnnData")
            if use_rep is None:
                if 'X_pca' in adata.obsm.keys():
                    logger.info(f"👀 Detected PCA coordinates in the object, will use these to calculate distances")
                    use_rep = 'X_pca'
                else:
                    logger.info(f"🧙 Using the expression matrix to calculate distances")
                    use_rep = 'X'
            elif (use_rep not in adata.obsm.keys()) and (use_rep != 'X'):
                raise KeyError(
                        f"🛑 '{use_rep}' is not found in `.obsm`")
            if use_rep == 'X' and adata.n_vars > 15000:
                logger.warn(f"⚠ Warning: {adata.n_vars} features are used for calculating distances. Subsetting the AnnData into HVGs is recommended")
            if metric is None:
                metric = 'correlation' if use_rep == 'X' else 'euclidean'
        Cell_X = adata.X if use_rep == 'X' else adata.obsm[use_rep]
        IDs = adata.obs_names
        datasets = adata.obs[dataset].astype(str).values
        celltypes = adata.obs[cell_type].astype(str).values
        use_Cell_X = Cell_X if use_rep != 'X' else np.expm1(Cell_X)
        Celltype_X = []
        col_ds = []
        col_cs =[]
        for d in np.unique(datasets):
            for c in np.unique(celltypes[datasets == d]):
                col_cs.append(c)
                col_ds.append(d)
                m = use_Cell_X[(datasets == d) & (celltypes == c), :].mean(axis = 0)
                Celltype_X.append(m.A1 if isinstance(m, np.matrix) else m)
        Celltype_X = np.log1p(np.array(Celltype_X)) if use_rep == 'X' else np.array(Celltype_X)
        if metric not in ['cityblock', 'cosine', 'euclidean', 'l1', 'l2', 'manhattan']:
            if isinstance(Cell_X, spmatrix):
                Cell_X = Cell_X.toarray()
            if isinstance(Celltype_X, spmatrix):
                Celltype_X = Celltype_X.toarray()
        dist_mat = pairwise_distances(Cell_X, Celltype_X, metric = metric, n_jobs = n_jobs, **kwargs)
        cell = pd.DataFrame(dict(dataset=datasets, ID=IDs, cell_type=celltypes))
        cell_type = pd.DataFrame(dict(dataset=col_ds, cell_type=col_cs))
        return Distance(dist_mat, cell, cell_type)

    def normalize(self, Gaussian_kernel: bool = False, rank: bool = True, normalize: bool = True) -> None:
        """
        Normalize the distance matrix with a Gaussian kernel.

        Parameters
        ----------
        Gaussian_kernel
            Whether to apply the Gaussian kernel to the distance matrix.
            (Default: `False`)
        rank
            Whether to turn the matrix into a rank matrx.
            (Default: `True`)
        normalize
            Whether to maximum-normalize the distance matrix.
            (Default: `True`)

        Returns
        ----------
        None
            The :class:`~cellhint.distance.Distance` object modified with a normalized distance matrix.
        """
        if Gaussian_kernel:
            sds = np.sqrt((self.dist_mat ** 2).sum(axis = 1) / self.n_cell_type)[:, np.newaxis]
            self.dist_mat = np.exp(- self.dist_mat / (2 / sds)**2)
            self.dist_mat = 1 - self.dist_mat / self.dist_mat.sum(axis = 1)[:, np.newaxis]
        if rank:
            self.dist_mat = rankdata(self.dist_mat).reshape(self.dist_mat.shape)
        if normalize:
            self.dist_mat = self.dist_mat / self.dist_mat.max()

    def concatenate(self, *distances, by: str = 'cell', check: bool = False):
        """
        Concatenate by either cells (rows) or cell types (columns).

        Parameters
        ----------
        distances
            A :class:`~cellhint.distance.Distance` object or a list of such objects.
        by
            The direction of concatenation, joining either cells (`'cell'`, rows) or cell types (`'cell_type'`, columns).
            (Default: `'cell'`)
        check
            Check whether the concatenation is feasible.
            (Default: `False`)

        Returns
        ----------
        :class:`~cellhint.distance.Distance`
            A :class:`~cellhint.distance.Distance` object concatenated along cells (`by = 'cell'`) or cell types (`by = 'cell_type'`).
        """
        distances = distances[0] if isinstance(distances[0], (list, tuple, set)) else distances
        distances = tuple(distances)
        all_distances = (self,) + distances
        if by not in ['cell', 'cell_type']:
            raise ValueError(
                    f"🛑 Unrecognized `by` value, should be one of `'cell'` or `'cell_type'`")
        if check:
            series_compare = [(x.cell_type.dataset+x.cell_type.cell_type).sort_values() for x in all_distances] if by == 'cell' else [(x.cell.dataset+x.cell.ID).sort_values() for x in all_distances]
            if pd.concat(series_compare, axis = 1).T.drop_duplicates().shape[0] > 1:
                raise Exception(
                        f"🛑 Concatenation is not feasible. Please ensure the meta-information is matched")
        if by == 'cell':
            dist_mat = np.concatenate([x.dist_mat for x in all_distances], axis = 0)
            cell = pd.concat([x.cell for x in all_distances], axis = 0, ignore_index = True)
            return Distance(dist_mat, cell, self.cell_type)
        else:
            match_base = (self.cell.dataset+self.cell.ID).reset_index().set_index(0)
            indices = [np.argsort(match_base.loc[x.cell.dataset+x.cell.ID, 'index'].values) for x in distances]
            dist_mat = np.concatenate([self.dist_mat] + [x.dist_mat[y, :] for x,y in zip(distances, indices)], axis = 1)
            cell_type = pd.concat([x.cell_type for x in all_distances], axis = 0, ignore_index = True)
            return Distance(dist_mat, self.cell, cell_type)

    def symmetric(self) -> bool:
        """
        Check whether the distance matrix is symmetric in terms of datasets and cell types.

        Returns
        ----------
        bool
            `True` or `False` indicating whether all datasets and cell types are included in the object (thus symmetric).
        """
        return np.array_equal(np.unique(self.cell.dataset + self.cell.cell_type), np.unique(self.cell_type.dataset + self.cell_type.cell_type))

    def filter_cells(self, check_symmetry: bool = True) -> None:
        """
        Filter out cells whose gene expression profiles do not correlate most with the eigen cell they belong to (i.e., correlate most with other cell types).

        Parameters
        ----------
        check_symmetry
            Whether to check the symmetry of the distance matrix in terms of datasets and cell types.
            (Default: `True`)

        Returns
        ----------
        None
            A :class:`~cellhint.distance.Distance` object with undesirable cells filtered out.
        """
        if check_symmetry and not self.symmetric():
            raise ValueError(
                    f"🛑 Cell filtering is not possible. Please provide the matrix with symmetric datasets and cell types")
        bool_cell = np.ones(self.n_cell, dtype=bool)
        for i, s in self.cell.iterrows():
            flag_dataset = self.cell_type.dataset == s['dataset']
            if self.cell_type.cell_type.values[flag_dataset][self.dist_mat[i][flag_dataset].argmin()] != s['cell_type']:
                bool_cell[i] = False
        if (~bool_cell).sum() == 0:
            logger.info(f"✂ No cells are filtered out")
        else:
            ds_unique, ds_table = np.unique(self.cell.dataset.values[~bool_cell], return_counts = True)
            if len(ds_unique) == 1:
                logger.info(f"✂ {(~bool_cell).sum()} cells are filtered out from {ds_unique[0]}")
            else:
                logger.info(f"✂ {(~bool_cell).sum()} cells are filtered out, including:")
                for m, n in zip(ds_unique, ds_table):
                    logger.info(f"      {n} cells from {m}")
            self.dist_mat = self.dist_mat[bool_cell]
            self.cell = self.cell[bool_cell]
            all_combine = (self.cell_type.dataset + ': ' + self.cell_type.cell_type).values
            left_combine = np.unique(self.cell.dataset + ': ' + self.cell.cell_type)
            if len(left_combine) < len(all_combine):
                column_keep = np.isin(all_combine, left_combine)
                self.dist_mat = self.dist_mat[:, column_keep]
                self.cell_type = self.cell_type[column_keep]
                logger.info(f"✂ The following cell types are discarded due to low confidence in annotation:")
                for rec in all_combine[~column_keep]:
                    logger.info(f"      {rec}")

    def to_meta(self, check_symmetry: bool = True, turn_binary: bool = False, return_symmetry: bool = True) -> pd.DataFrame:
        """
        Meta-analysis of cross-dataset cell type dissimilarity or membership.

        Parameters
        ----------
        check_symmetry
            Whether to check the symmetry of the distance matrix in terms of datasets and cell types.
            (Default: `True`)
        turn_binary
            Whether to turn the distance matrix into a cell type membership matrix before meta analysis.
            (Default: `False`)
        return_symmetry
            Whether to return a symmetric dissimilarity matrix by averaging with its transposed form.
            (Default: `True`)

        Returns
        ----------
        :class:`~pandas.DataFrame`
            A :class:`~pandas.DataFrame` object representing the cell-type-level dissimilarity matrix (`turn_binary = False`) or membership matrix (`turn_binary = True`).
        """
        if check_symmetry and not self.symmetric():
            raise ValueError(
                    f"🛑 Meta cell analysis is not possible. Concatenate all datasets and cell types beforehand using `concatenate`")
        use_mat = self.to_binary(False if check_symmetry else True).dist_mat if turn_binary else self.dist_mat
        meta_cell = []
        for _, s in self.cell_type.iterrows():
            meta_cell.append(use_mat[(self.cell.dataset == s['dataset']) & (self.cell.cell_type == s['cell_type']), :].mean(axis = 0))
        meta_cell = pd.DataFrame(np.array(meta_cell))
        meta_cell.index = (self.cell_type.dataset + ': ' + self.cell_type.cell_type).values
        meta_cell.columns = meta_cell.index
        return (meta_cell + meta_cell.T)/2 if return_symmetry else meta_cell

    def to_binary(self, check_symmetry: bool = True):
        """
        Turn the distance matrix into a binary matrix representing the estimated cell type membership across datasets.

        Parameters
        ----------
        check_symmetry
            Whether to check the symmetry of the distance matrix in terms of datasets and cell types.
            (Default: `True`)

        Returns
        ----------
        :class:`~cellhint.distance.Distance`
            A :class:`~cellhint.distance.Distance` object representing the estimated cell type membership across datasets.
        """
        if check_symmetry and not self.symmetric():
            raise ValueError(
                    f"🛑 Cannot convert to a binary matrix. Please provide the matrix with symmetric datasets and cell types")
        member_mat = np.zeros(self.shape, dtype = int)
        datasets = self.cell_type.dataset.values
        for dataset in np.unique(datasets):
            indices = np.where(datasets == dataset)[0]
            member_mat[range(member_mat.shape[0]), indices[self.dist_mat[:, indices].argmin(axis = 1)]] = 1
        return Distance(member_mat, self.cell, self.cell_type)

    def assign(self) -> None:
        """
        Assign each cell to its most similar cell type in each dataset.

        Returns
        ----------
        None
            Modified object with the result of cell assignment added as `.assignment`.
        """
        assignment = {}
        for dataset in np.unique(self.cell_type.dataset):
            flag = self.cell_type.dataset == dataset
            assignment[dataset] = self.cell_type.cell_type.values[flag][self.dist_mat[:, flag].argmin(axis = 1)]
        assignment = pd.DataFrame(assignment, index = self.cell.index)
        #no need to assign cells for the dataset they belong to
        for dataset in assignment.columns:
            flag = self.cell.dataset == dataset
            assignment.loc[flag, dataset] = self.cell.cell_type.values[flag]
        self.assignment = assignment

    def to_confusion(self, D1: str, D2: str, check: bool = True) -> tuple:
        """
        This function is deprecated. Use `to_pairwise_confusion` and `to_multi_confusion` instead.
        Extract the dataset1-by-dataset2 and dataset2-by-dataset1 confusion matrices. Note this function is expected to be applied to a binary membership matrix.

        Parameters
        ----------
        D1
            Name of the first dataset.
        D2
            Name of the second dataset.
        check
            Whether to check names of the two datasets are contained.
            (Default: `True`)

        Returns
        ----------
        tuple
            The dataset1-by-dataset2 and dataset2-by-dataset1 confusion matrices.
        """
        if check and not {D1, D2}.issubset(np.unique(self.cell_type.dataset)):
            raise ValueError(
                    f"🛑 Please provide correct dataset names")
        D1_col_flag = self.cell_type.dataset == D1
        D2_col_flag = self.cell_type.dataset == D2
        D1_celltypes = self.cell_type.cell_type.values[D1_col_flag]
        D2_celltypes = self.cell_type.cell_type.values[D2_col_flag]
        D1_row_flag = self.cell.dataset == D1
        D2_row_flag = self.cell.dataset == D2
        D1byD2 = pd.DataFrame(np.array([self.dist_mat[D1_row_flag & (self.cell.cell_type == x)][:, D2_col_flag].sum(axis=0) for x in D1_celltypes]), columns = D2_celltypes, index = D1_celltypes)
        D2byD1 = pd.DataFrame(np.array([self.dist_mat[D2_row_flag & (self.cell.cell_type == x)][:, D1_col_flag].sum(axis=0) for x in D2_celltypes]), columns = D1_celltypes, index = D2_celltypes)
        return D1byD2, D2byD1

    def to_pairwise_confusion(self, D1: str, D2: str, check: bool = True) -> tuple:
        """
        Extract the dataset1-by-dataset2 and dataset2-by-dataset1 confusion matrices.

        Parameters
        ----------
        D1
            Name of the first dataset.
        D2
            Name of the second dataset.
        check
            Whether to check names of the two datasets are contained.
            (Default: `True`)

        Returns
        ----------
        tuple
            The dataset1-by-dataset2 and dataset2-by-dataset1 confusion matrices.
        """
        if check and not {D1, D2}.issubset(np.unique(self.cell_type.dataset)):
            raise ValueError(
                    f"🛑 Please provide correct dataset names")
        if not hasattr(self, 'assignment'):
            raise AttributeError(
                    f"🛑 No `.assignment` attribute in the object. Use the `.assign` method first")
        D1_flag = (self.cell.dataset == D1)
        D2_flag = (self.cell.dataset == D2)
        D1byD2 = pd.crosstab(self.cell.cell_type[D1_flag], self.assignment.loc[D1_flag, D2])
        D2byD1 = pd.crosstab(self.cell.cell_type[D2_flag], self.assignment.loc[D2_flag, D1])
        D1byD2_lack_columns = D2byD1.index.difference(D1byD2.columns)
        if len(D1byD2_lack_columns) > 0:
            D1byD2 = D1byD2.join(pd.DataFrame(np.zeros((len(D1byD2.index), len(D1byD2_lack_columns)), dtype=int), index = D1byD2.index, columns = D1byD2_lack_columns))
        D2byD1_lack_columns = D1byD2.index.difference(D2byD1.columns)
        if len(D2byD1_lack_columns) > 0:
            D2byD1 = D2byD1.join(pd.DataFrame(np.zeros((len(D2byD1.index), len(D2byD1_lack_columns)), dtype=int), index = D2byD1.index, columns = D2byD1_lack_columns))
        return D1byD2, D2byD1.loc[D1byD2.columns, D1byD2.index]

    def to_multi_confusion(self, relation: pd.DataFrame, D: str, check: bool = True) -> tuple:
        """
        Extract the confusion matrices between meta-cell-types defined prior and cell types from a new dataset.

        Parameters
        ----------
        relation
            A :class:`~pandas.DataFrame` object representing the cell type harmonization result across multiple datasets.
        D
            Name of the new dataset to be aligned.
        check
            Whether to check names of the datasets are contained.
            (Default: `True`)

        Returns
        ----------
        tuple
            The confusion matrices between meta-cell-types defined prior and cell types from a new dataset.
        """
        datasets = relation.columns[0::2]
        if check:
            if not set(datasets).issubset(np.unique(self.cell_type.dataset)):
                raise ValueError(
                        f"🛑 `relation` contains unexpected dataset names")
            if D not in np.unique(self.cell_type.dataset) or D in datasets:
                raise ValueError(
                        f"🛑 Please provide a valid dataset name `D`")
        if not hasattr(self, 'assignment'):
            raise AttributeError(
                    f"🛑 No `.assignment` attribute in the object. Use the `.assign` method first")
        #D1byD2
        D1_flag = self.cell.dataset.isin(datasets)
        D1_assign = self.assignment[D1_flag]
        D1_truth = np.full(D1_assign.shape[0], UNASSIGN, dtype = object)
        for _, s in relation.iterrows():
            celltypes = s.values[0::2]
            non_blank_flag = ~np.isin(celltypes, [NOVEL, REMAIN])
            existing_datasets = datasets[non_blank_flag]
            existing_celltypes = celltypes[non_blank_flag]
            flag = np.all(D1_assign[existing_datasets] == existing_celltypes, axis = 1).values & self.cell[D1_flag].dataset.isin(existing_datasets).values
            D1_truth[flag] = ' '.join(s.values)
        D1_used = D1_truth != UNASSIGN
        D1byD2 = pd.crosstab(D1_truth[D1_used], D1_assign.loc[D1_used, D])
        #D2byD1
        D2_flag = self.cell.dataset == D
        D2_assign = self.assignment[D2_flag]
        D2_predict = np.full(D2_assign.shape[0], UNASSIGN, dtype = object)
        for _, s in relation.iterrows():
            celltypes = s.values[0::2]
            flags = (D2_assign[datasets] == celltypes) | np.isin(celltypes, [NOVEL, REMAIN])
            D2_predict[np.all(flags, axis = 1).values] = ' '.join(s.values)
        D2_used = D2_predict != UNASSIGN
        D2byD1 = pd.crosstab(self.cell.cell_type[D2_flag][D2_used], D2_predict[D2_used])
        #warning
        if relation.shape[0] > D1byD2.shape[0]:
            lost_celltypes = np.setdiff1d(relation.apply(lambda row: ' '.join(row.values), axis = 1).values, D1byD2.index)
            logger.warn(f"⚠ Warning: no cells are found to match these patterns: {set(lost_celltypes)}. Double check the harmonized relationships before integrating '{D}'")
            D1byD2 = pd.concat([D1byD2, pd.DataFrame(np.zeros((len(lost_celltypes), len(D1byD2.columns)), dtype=int), index = lost_celltypes, columns = D1byD2.columns)], axis = 0)
        #a unique cell type in D2 may be annotated to nothing and filtered
        lost_celltypes = np.setdiff1d(np.unique(self.cell.cell_type[D2_flag]), D2byD1.index)
        if len(lost_celltypes) > 0:
            D2byD1 = pd.concat([D2byD1, pd.DataFrame(np.zeros((len(lost_celltypes), len(D2byD1.columns)), dtype=int), index = lost_celltypes, columns = D2byD1.columns)], axis = 0)
        #return
        D1byD2_lack_columns = D2byD1.index.difference(D1byD2.columns)
        if len(D1byD2_lack_columns) > 0:
            D1byD2 = D1byD2.join(pd.DataFrame(np.zeros((len(D1byD2.index), len(D1byD2_lack_columns)), dtype=int), index = D1byD2.index, columns = D1byD2_lack_columns))
        D2byD1_lack_columns = D1byD2.index.difference(D2byD1.columns)
        if len(D2byD1_lack_columns) > 0:
            D2byD1 = D2byD1.join(pd.DataFrame(np.zeros((len(D2byD1.index), len(D2byD1_lack_columns)), dtype=int), index = D2byD1.index, columns = D2byD1_lack_columns))
        return D1byD2, D2byD1.loc[D1byD2.columns, D1byD2.index]


"""
.. testsetup:: *

    from zasim.cagen.utils import *

"""
# This file is part of zasim. zasim is licensed under the BSD 3-clause license.
# See LICENSE.txt for details.

from ..features import HAVE_TUPLE_ARRAY_INDEX

from itertools import product
import numpy as np

if HAVE_TUPLE_ARRAY_INDEX:
    def offset_pos(pos, offset):
        """Offset a position by an offset. Any amount of dimensions should work.

        >>> offset_pos((1, ), (5, ))
        (6,)
        >>> offset_pos((1, 2, 3), (9, 8, 7))
        (10, 10, 10)"""
        if len(pos) == 1:
            return (pos[0] + offset[0],)
        else:
            return tuple([a + b for a, b in zip(pos, offset)])
else:
    def offset_pos(pos, offset):
        """Offset a position by an offset. Only works for 1d."""
        if isinstance(pos, tuple):
            pos = pos[0]
        if isinstance(offset, tuple):
            offset = offset[0]
        return pos + offset

def gen_offset_pos(pos, offset):
    """Generate code to offset a position by an offset.

    >>> gen_offset_pos(["i", "j"], ["foo", "bar"])
    ['i + foo', 'j + bar']"""
    return ["%s + %s" % (a, b) for a, b in zip(pos, offset)]

def dedent_python_code(code):
    '''
    Dedent a bit of python code, like this:

    >>> print dedent_python_code("""# update the histogram
    ...     if result != center:
    ...         self.target.histogram[result] += 1""")
    # update the histogram
    if result != center:
        self.target.histogram[result] += 1
    '''

    lines = code.split("\n")
    resultlines = [lines[0]] # the first line shall never have any whitespace.
    if len(lines) > 1:
        common_whitespace = len(lines[1]) - len(lines[1].lstrip())
        if common_whitespace > 0:
            for line in lines[1:]:
                white, text = line[:common_whitespace], line[common_whitespace:]
                assert line == "" or white.isspace()
                resultlines.append(text)
        else:
            resultlines.extend(lines[1:])
    return "\n".join(resultlines)

def rule_nr_to_multidim_rule_arr(number, digits, base=2):
    """Given the rule `number`, the number of cells the neighbourhood has
    (as `digits`) and the `base` of the cells, this function calculates the
    multidimensional rule table for computing that rule."""

    if base < 256: dtype = "int8"
    else: dtype = "int16" # good luck with that.

    res = np.zeros((base,) * digits, dtype=dtype)
    entries = base ** digits
    blubb = base ** entries
    for position in product(*([xrange(base-1, -1, -1)] * digits)):
        blubb /= base
        d = int(number // (blubb))
        number -= d * (blubb)
        res[position] = d

    return res

def rule_nr_to_rule_arr(number, digits, base=2):
    """Given a rule `number`, the number of cells the neighbourhood has
    (as `digits`) and the `base` of the cells, this function calculates the
    lookup array for computing that rule.

    >>> rule_nr_to_rule_arr(110, 3)
    [0, 1, 1, 1, 0, 1, 1, 0]
    >>> rule_nr_to_rule_arr(26, 3, 3)
    [2, 2, 2, ...]
    """
    entries = base ** digits
    result = [0 for index in range(entries)]
    blubb = base ** entries
    for e in range(entries - 1, -1, -1):
        blubb /= base
        d = int(number // (blubb))
        number -= d * (blubb)
        result[e] = d

    return result

def elementary_digits_and_values(neighbourhood, base=2, rule_arr=None):
    """From a neighbourhood, the base of the values used and the array that
    holds the results for each combination of neighbourhood values, create a
    list of dictionaries with the neighbourhood values paired with their
    result_value ordered by the position like in the rule array.

    If the rule_arr is None, no result_value field will be generated."""
    digits_and_values = []
    offsets = neighbourhood.offsets
    names = neighbourhood.names
    digits = len(offsets)

    for i in range(base ** digits):
        values = rule_nr_to_rule_arr(i, digits, base)
        asdict = dict(zip(names, values))
        digits_and_values.append(asdict)

    if rule_arr is not None:
        if not isinstance(rule_arr, np.ndarray) or len(rule_arr.shape) == 1:
            indices = enumerate(xrange(base ** digits))
        else:
            indices = enumerate(reversed(list(product(*([xrange(base-1,-1,-1)] * digits)))))

        for index, rule_idx in indices:
            digits_and_values[index].update(result_value = rule_arr[rule_idx])
    return digits_and_values


from django.urls import path

from . import views

app_name = 'blog'

urlpatterns = [
    path('', views.index, name="index"),
    path('detalhes/<int:pk>/<slug:slug>', views.detail, name="details"),
    path('post/novo/', views.post, name="new_post"),
    path('editar/post/<int:pk>', views.edit, name="edit"),
    path('filter/<int:pk>/<str:username>', views.filter, name="filter"),
    path('search/', views.search, name="search"),
    path('delete/<int:pk>', views.delete, name="delete_post"),
    path('contato/', views.contato, name="contato"),
]


import tkinter as tk
import sqlite3

def guardar_palabras():
    palabras = [entrada1.get(), entrada2.get(), entrada3.get(), entrada4.get(), entrada5.get()]

    # Conexión a la base de datos
    conexion = sqlite3.connect('basedatos.db')
    cursor = conexion.cursor()

    # Crear la tabla "palabras" si no existe
    cursor.execute('''CREATE TABLE IF NOT EXISTS palabras
                      (id INTEGER PRIMARY KEY AUTOINCREMENT,
                       palabra TEXT)''')

    # Eliminar las palabras anteriores en la tabla
    cursor.execute("DELETE FROM palabras")

    # Insertar las últimas 5 palabras en la tabla "palabras"
    for palabra in palabras:
        cursor.execute("INSERT INTO palabras (palabra) VALUES (?)", (palabra,))

    # Guardar cambios y cerrar conexión
    conexion.commit()
    conexion.close()
    ventana.destroy()

ventana = tk.Tk()
ventana.title("5 Palabras")
frase_inicio = "Si fueras 5 palabras, cuáles serías?:"
etiqueta_frase = tk.Label(ventana, text=frase_inicio)
etiqueta_frase.pack()
entrada1 = tk.Entry(ventana)
entrada1.pack()
entrada2 = tk.Entry(ventana)
entrada2.pack()
entrada3 = tk.Entry(ventana)
entrada3.pack()
entrada4 = tk.Entry(ventana)
entrada4.pack()
entrada5 = tk.Entry(ventana)
entrada5.pack()
boton = tk.Button(ventana, text="Aceptar", command=guardar_palabras)
boton.pack()
ventana.mainloop()

def mostrar_ventana1():
    ventana1 = tk.Toplevel()
    ventana1.title("Tus palabras")
    etiqueta1 = tk.Label(ventana1, text="Estas son tus palabras:")
    etiqueta1.pack()

    # Conexión a la base de datos
    conexion = sqlite3.connect('basedatos.db')
    cursor = conexion.cursor()

    # Consulta para recuperar las palabras
    cursor.execute("SELECT palabra FROM palabras")
    palabras = cursor.fetchall()

    for palabra in palabras:
        etiqueta = tk.Label(ventana1, text=palabra[0])
        etiqueta.pack()

    # Cerrar conexión
    conexion.close()

def mostrar_ventana2():
    ventana2 = tk.Toplevel()
    ventana2.title("Palabras nuevas")
    frase_inicio = "Si fueras 5 palabras, cuáles serías?:"
    etiqueta_frase = tk.Label(ventana2, text=frase_inicio)
    etiqueta_frase.pack()

    entrada1 = tk.Entry(ventana2)
    entrada1.pack()
    entrada2 = tk.Entry(ventana2)
    entrada2.pack()
    entrada3 = tk.Entry(ventana2)
    entrada3.pack()
    entrada4 = tk.Entry(ventana2)
    entrada4.pack()
    entrada5 = tk.Entry(ventana2)
    entrada5.pack()

    def guardar_palabras_nuevas():
        palabras = [entrada1.get(), entrada2.get(), entrada3.get(), entrada4.get(), entrada5.get()]

        # Conexión a la base de datos
        conexion = sqlite3.connect('basedatos.db')
        cursor = conexion.cursor()

        # Crear la tabla "palabras" si no existe
        cursor.execute('''CREATE TABLE IF NOT EXISTS palabras
                            (id INTEGER PRIMARY KEY AUTOINCREMENT,
                             palabra TEXT)''')

        # Eliminar las palabras anteriores en la tabla
        cursor.execute("DELETE FROM palabras")

        # Insertar las últimas 5 palabras en la tabla "palabras"
        for palabra in palabras:
            cursor.execute("INSERT INTO palabras (palabra) VALUES (?)", (palabra,))

        # Guardar cambios y cerrar conexión
        conexion.commit()
        conexion.close()

    boton = tk.Button(ventana2, text="Aceptar", command=guardar_palabras_nuevas)
    boton.pack()

ventana_principal = tk.Tk()
ventana_principal.title("Ventana Principal")

boton_ventana1 = tk.Button(ventana_principal, text="Tus palabras", command=mostrar_ventana1)
boton_ventana1.pack()

boton_ventana2 = tk.Button(ventana_principal, text="Palabras nuevas", command=mostrar_ventana2)
boton_ventana2.pack()

ventana_principal.mainloop()


import datetime
import sqlite3
import os
import sys
from PyQt6.QtWidgets import *
from PyQt6.QtCore import Qt
from docxtpl import DocxTemplate

class mailbackGenWindow(QMainWindow):
    def __init__(self):
        super().__init__()

        self.setWindowTitle("Test Mailback Letter Generator")
        self.setFixedSize(722, 479)

        main_layout = QVBoxLayout()

        self.db = sqlite3.connect("test_mailback.db")
        self.cur = self.db.cursor()

        client_label = QLabel("Select Client: ")
        self.client_select = QComboBox()
        self.populateClientSelect()

        def setAndGet():
            self.getDefaultAddress()
            self.setDefaultAddress()

        self.client_select.currentIndexChanged.connect(setAndGet)

        reason_label = QLabel("Select All Reasons for Return ")
        self.reason_select = QFrame()
        self.reason_layout = QGridLayout()
        self.reasonCheckBoxList = []
        self.address1 = QLineEdit()
        self.address1.setFixedWidth(200)
        self.address2 = QLineEdit()
        self.address2.setFixedWidth(200)
        self.address3 = QLineEdit()
        self.address3.setFixedWidth(200)

        self.clear_address_button = QPushButton("Clear Address")
        self.clear_address_button.clicked.connect(self.clearAddress)
        self.default_address_button = QPushButton("Default")
        self.default_address_button.clicked.connect(self.setDefaultAddress)
        self.populateReasonLayout()
        self.reason_select.setLayout(self.reason_layout)

        self.reason_error = QLabel("Please select at least one reason.")
        self.reason_error.setStyleSheet("color: red")
        self.reason_error.hide()

        self.envelope_button = QPushButton("Generate Envelope")
        self.envelope_button.clicked.connect(self.printEnvelope)

        self.large_envelope_button = QPushButton("Large Envelope Sheet")
        self.large_envelope_button.clicked.connect(self.printLargeEnvelope)

        self.submit_button = QPushButton("Generate Letter")
        self.submit_button.clicked.connect(self.generateLetter)

        widgets = [client_label, self.client_select, reason_label, self.reason_select, self.reason_error,
                   self.submit_button, self.envelope_button, self.large_envelope_button]

        for w in widgets:
            main_layout.addWidget(w)
        widget = QWidget()
        widget.setLayout(main_layout)

        # Set the central widget of the Window. Widget will expand
        # to take up all the space in the window by default.
        self.setCentralWidget(widget)
        self.template = DocxTemplate("test_mailback_template.docx")

        self.envelope = DocxTemplate("mailout.docx")

        self.big_envelope = DocxTemplate("large envelope template.docx")

        self.current_date = datetime.date.today().strftime('%m/%d/%Y')

        self.currentClient = ""
        self.currentAddr1 = ""
        self.currentAddr2 = ""
        self.currentPhoneNumber = ""
        self.getDefaultAddress()
        self.setDefaultAddress()

    def populateClientSelect(self):
        tups = self.cur.execute("""SELECT query_name FROM client
                                ORDER BY query_name ASC;""")

        clients = [name for t in tups for name in t]
        self.client_select.addItems(clients)

    def getDefaultAddress(self):
        client_name = self.client_select.currentText()
        client_row = self.cur.execute("""SELECT full_name, address, phone_number
                                                    FROM client
                                                    WHERE query_name = ?""", (client_name,))
        self.currentClient, full_addr, self.currentPhoneNumber = [c for t in client_row for c in t]
        self.currentAddr1, self.currentAddr2 = full_addr.split('*')

    def setDefaultAddress(self):
        self.address1.setText(self.currentClient)
        self.address2.setText(self.currentAddr1)
        self.address3.setText(self.currentAddr2)

    def clearAddress(self):
        self.address1.clear()
        self.address2.clear()
        self.address3.clear()

    def populateReasonLayout(self):
        reasonTypes = self.cur.execute("""SELECT DISTINCT type FROM mailback_reason;""")
        reasonTypes = [t for rt in reasonTypes for t in rt]
        print(reasonTypes)

        column = 0
        row = 0
        for t in reasonTypes:
            if column == 2:
                column = 0
                row += 1

            frame = QFrame()
            layout = QVBoxLayout()
            layout.addWidget(QLabel(t + ':'))
            reasons = self.cur.execute("""SELECT reason FROM mailback_reason
                                            WHERE type = ?;""", (t,))
            reasons = [r for rt in reasons for r in rt]
            for r in reasons:
                box = QCheckBox(r)
                self.reasonCheckBoxList.append(box)
                layout.addWidget(box)
            frame.setLayout(layout)
            self.reason_layout.addWidget(frame, column, row, Qt.AlignmentFlag.AlignTop)

            column += 1

        if column == 2:
            column = 0
            row += 1

        frame = QFrame()
        layout = QGridLayout()
        layout.addWidget(QLabel('Name:'), 0, 0, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(self.address1, 0, 1, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(QLabel('Address:'), 1, 0, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(self.address2, 1, 1, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(QLabel('City/State/Zip:'), 2, 0, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(self.address3, 2, 1, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(self.clear_address_button, 3, 0, Qt.AlignmentFlag.AlignLeft)
        layout.addWidget(self.default_address_button, 3, 1, Qt.AlignmentFlag.AlignLeft)
        frame.setLayout(layout)
        self.reason_layout.addWidget(frame, column, row, Qt.AlignmentFlag.AlignLeft)

    def generateLetter(self):
        #FOR SETTING FIXED WIDTH/HEIGHT
        #print(self.width())
        #print(self.height())

        # avoids Microsoft Word opening dialog box saying that letter.docx caused error
        if os.path.exists("letter.docx"):
            os.remove("letter.docx")

        reasons = []

        for box in self.reasonCheckBoxList:
            if box.isChecked():
                reasons.append(box.text())
                box.setChecked(False)

        reason = ""
        rlength = len(reasons)
        if rlength == 1:
            reason = reasons[0]
        elif rlength == 2:
            reason = reasons[0] + ' and ' + reasons[1]
        elif rlength > 2:
            for i in range(0, rlength):
                if i != rlength - 1:
                    reason += reasons[i] + ', '
                else:
                    reason += 'and ' + reasons[i]
        else:  # reasons is empty
            self.reason_error.show()
            return 1

        self.reason_error.hide()
        self.submit_button.setEnabled(False)

        fill_in = {"date": self.current_date,
                   "client": self.currentClient,
                   "reason": reason,
                   "address_1": self.currentAddr1,
                   "address_2": self.currentAddr2,
                   "phone_number": self.currentPhoneNumber
                   }

        self.template.render(fill_in)
        self.template.save('letter.docx')
        os.startfile("letter.docx", "print")

        self.submit_button.setEnabled(True)

    def printEnvelope(self):
        self.envelope_button.setEnabled(False)

        fill_in = {"client": self.address1.text(),
                   "addr_1": self.address2.text(),
                   "addr_2": self.address3.text()}

        self.envelope.render(fill_in)
        self.envelope.save('envelope.docx')
        os.startfile("envelope.docx", "print")

        self.envelope_button.setEnabled(True)

    def printLargeEnvelope(self):
        self.large_envelope_button.setEnabled(False)

        fill_in = {"client": self.address1.text(),
                   "addr_1": self.address2.text(),
                   "addr_2": self.address3.text()}

        self.big_envelope.render(fill_in)
        self.big_envelope.save('big_envelope.docx')
        os.startfile("big_envelope.docx", "print")

        self.large_envelope_button.setEnabled(True)

def main():
    app = QApplication(sys.argv)
    window = mailbackGenWindow()
    window.show()

    app.exec()

if __name__ == "__main__":
    main()