"""
Fine-tuning the library models for causal language modeling (llama, opt, ...)
on a text file or a dataset without using HuggingFace Trainer.
"""

import logging
import math
import os
import shutil

import tensorflow.compat.v2 as tf
import torch
import transformers
from accelerate import Accelerator
from accelerate.logging import get_logger
from accelerate.utils import recursively_apply, set_seed
from peft import LoraConfig, inject_adapter_in_model, prepare_model_for_kbit_training
from torch.nn import functional as F
from tqdm.auto import tqdm
from transformers import (
    MODEL_MAPPING,
    AutoConfig,
    AutoModelForCausalLM,
    BitsAndBytesConfig,
    get_scheduler,
)
from utils import data_utils as dutils
from utils.args import parse_args

from neurocache import NeurocacheModelForCausalLM, OnDeviceCacheConfig
from neurocache.neurocache import get_attribute

# Make sure that tensorflow is not reserving GPUs.
tf.config.experimental.set_visible_devices([], "GPU")

logger = get_logger(__name__)

MODEL_CONFIG_CLASSES = list(MODEL_MAPPING.keys())

def init_neurocache_weights_(args, neurocache):
    _model = AutoModelForCausalLM.from_pretrained(
        args.model_name_or_path,
        config=AutoConfig.from_pretrained(
            args.model_name_or_path,
        ),
    )

    if hasattr(_model, "base_model"):
        _model = _model.model
    if hasattr(_model, "model"):
        _model = _model.model
    if hasattr(_model, "decoder"):
        _model = _model.decoder
    layers = get_attribute(_model, "layers")

    for i, idx in enumerate(neurocache.config.attention_layers):
        self_attn_module = get_attribute(layers[idx], "self_attn")
        neurocache.cache_attns[i]._init_proj_weights(self_attn_module)

    del _model

def prevent_full_backward_pass(_model):
    """
    Allow gradient checkpointing while preventing full backward pass on the model.
    """

    # Prevent full backward pass on the model. This is a workaround for the issue that
    # the full backward pass on the model will cause the memory and GPU usage to increase.

    # This is required because the default implementation of transformers does checkpointing
    # on all of the layers by default. This cause the backward pass to be done on all of the
    # layers, which is not necessary for our use case.

    # We fix this by enabling gradient checkpointing on the model, and then detaching the
    # output of the bottom layer. This will prevent the backward pass from propagating to
    # the bottom layer, and thus prevent the full backward pass on the model.

    # Note: this does not work by default due to PyTorch checkpoint function default use_reentrant
    # parameter to be True. We modify it to false in the transformers library.
    # modeling_utils.py:
    # After this line:
    # from torch.utils.checkpoint import checkpoint
    # Add this line:
    # checkpoint = functools.partial(checkpoint, use_reentrant=False)

    neurocache_config = _model.base_cache.config
    bottom_layer_idx = min(neurocache_config.attention_layers + neurocache_config.cache_layers)

    if hasattr(_model, "base_model"):
        _model = _model.model
    if hasattr(_model, "model"):
        _model = _model.model
    if hasattr(_model, "decoder"):
        _model = _model.decoder
    layers = get_attribute(_model, "layers")

    def forward_hook(module, args, output):
        return tuple(a.detach() if isinstance(a, torch.Tensor) else a for a in output)

    bottom_layer = layers[bottom_layer_idx - 1]
    bottom_layer.register_forward_hook(forward_hook)

def initialize_model(args, accelerator):
    if args.nf4:
        compute_dtype = getattr(torch, args.cache_dtype)
        quant_config = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type="nf4",
            bnb_4bit_compute_dtype=compute_dtype,
            device_map="auto",
        )
    else:
        quant_config = None

    config = AutoConfig.from_pretrained(
        args.model_name_or_path,
    )
    config.use_cache = False
    config._flash_attn_2_enabled = True

    model = AutoModelForCausalLM.from_pretrained(
        args.model_name_or_path,
        config=config,
        quantization_config=quant_config,
        torch_dtype=getattr(torch, args.cache_dtype) if not args.nf4 else None,
    )

    # Prepare the model for kbit training.
    if args.nf4:
        model = prepare_model_for_kbit_training(model)

    # Set up the neurocache config and wrap the model with the neurocache model.
    if args.cache_layers is not None:
        cache_layers = [int(x) for x in args.cache_layers.split(",")]
    else:
        cache_layers = [model.config.num_hidden_layers * 3 // 4]

    if args.attention_layers is not None:
        attention_layers = [int(x) for x in args.attention_layers.split(",")]
    else:
        attention_layers = list(range(min(cache_layers), model.config.num_hidden_layers))

    if not args.disable_lora:
        # Apply LoRA to the main model to adapt it to using neurocache.
        lora_layers = attention_layers
        lora_config = LoraConfig(
            r=args.lora_r,
            lora_alpha=args.lora_alpha,
            target_modules=args.lora_modules.split(","),
            lora_dropout=args.lora_dropout,
            layers_to_transform=lora_layers,
            bias="none",
            task_type="CAUSAL_LM",
        )
        logger.info(f"LoRA Config: {lora_config}")
        model = inject_adapter_in_model(lora_config, model, "neurocache")
    else:
        # freeze the model
        for param in model.parameters():
            param.requires_grad = False

    if not args.disable_neurocache:
        if not args.pretrained_neurocache:
            neurocache_config = OnDeviceCacheConfig(
                attention_layers=attention_layers,
                cache_layers=cache_layers,
                cache_size=args.cache_size,
                cache_dtype=args.cache_dtype,
                compression_factor=args.compression_factor,
                neighborhood_size=args.neighborhood_size,
                context_size=args.context_size,
                topk=args.topk,
                cache_type=args.cache_type,
            )
            model = NeurocacheModelForCausalLM(model, neurocache_config)
            logger.info(f"Neurocache Config: {neurocache_config}")
            logger.info("Initializing neurocache weights from pretrained model.")
            init_neurocache_weights_(args, model.base_cache)
        else:
            logger.info(f"Loading neurocache from {args.pretrained_neurocache}")
            model = NeurocacheModelForCausalLM.from_pretrained(
                model, args.pretrained_neurocache, cache_type="ONDEVICE", is_training=True
            )
        model = model.to(accelerator.device)

    if args.disable_grad_checkpointing:
        model.base_model.gradient_checkpointing_disable()
    else:
        model.base_model.gradient_checkpointing_enable()
        model.base_model.enable_input_require_grads()
        prevent_full_backward_pass(model)

    print_trainable_parameters(model, accelerator)
    return model

def calculate_metrics(eval_loss):
    try:
        perplexity = math.exp(eval_loss)
    except OverflowError:
        perplexity = float("inf")
    bits_per_token = eval_loss * 1.442695
    return bits_per_token, perplexity

def print_trainable_parameters(model, accelerator):
    """
    Prints the number of trainable parameters in the model.
    """
    trainable_params = 0
    all_param = 0
    for _, param in model.named_parameters():
        all_param += param.numel()
        if param.requires_grad:
            accelerator.print(_, param.shape)
            trainable_params += param.numel()
    accelerator.print(
        f"trainable params: {trainable_params} || all params: {all_param} "
        f"|| trainable%: {100 * trainable_params / all_param}"
    )

def initialize_dataloader(batch_size, task_config, split, accelerator, resume_step=0):
    distributed_batch_size = batch_size * accelerator.num_processes
    per_device_batch_size = batch_size

    if accelerator.is_local_main_process:
        dataset = dutils.LongTextDataset(
            task_config, split, distributed_batch_size, skip=resume_step
        )
    else:
        dataset = dutils.PlaceholderDataset(
            task_config, split, distributed_batch_size, skip=resume_step
        )

    dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, pin_memory=True)

    def slice_fn(data, tensor_slice, process_index=None, num_processes=None):
        """split batch across processes."""
        nonlocal per_device_batch_size

        def _slice_tensor(tensor, tensor_slice):
            # remove the additional batch dim that is added by the dataloader
            return tensor.squeeze(0)[tensor_slice]

        tensor_slice = slice(
            process_index * per_device_batch_size, (process_index + 1) * per_device_batch_size
        )
        return recursively_apply(_slice_tensor, data, tensor_slice)

    # Prepare dataloaders
    dataloader = accelerator.prepare_data_loader(dataloader, slice_fn_for_dispatch=slice_fn)
    return dataloader

def run_evaluation(args, model, dataloader, accelerator, global_step=0, prefix="eval"):
    logger.info("***** Running evaluation *****")
    logger.info(f"    Steps: {global_step}")
    logger.info(f"    Prefix: {prefix}")

    if not args.disable_neurocache:
        if isinstance(model, NeurocacheModelForCausalLM):
            model.base_cache.reinitialize_cache()
        elif isinstance(model, torch.nn.parallel.DistributedDataParallel):
            model.module.base_cache.reinitialize_cache()

    model.eval()
    losses = []
    progress_bar = tqdm(
        range(args.max_eval_steps if args.max_eval_steps > 0 else None),
        disable=not accelerator.is_local_main_process,
    )

    for step, batch in enumerate(dataloader):
        with torch.no_grad():
            target_ids = batch.pop("labels")
            epoch = batch.pop("epochs")[0].item()

            if args.disable_neurocache:
                batch.pop("start_of_sequence")

            outputs = model(**batch)
            loss = F.cross_entropy(
                outputs.logits.view(-1, outputs.logits.size(-1)).to(torch.float32),
                target_ids.view(-1),
                ignore_index=0,  # TODO: get this id from tokenizer
            )

        losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))
        if step >= args.max_eval_steps and args.max_eval_steps > 0:
            break

        if epoch > 0:
            logger.warning("Epoch > 0 for evaluation, this is unexpected.")

        progress_bar.update(1)

    eval_loss = torch.cat(losses).mean().item()
    bits_per_token, perplexity = calculate_metrics(eval_loss)

    logger.info(
        f"global_step: {global_step}, perplexity: {perplexity}, loss: {eval_loss}, bits_per_token: {bits_per_token}"
    )

    if args.with_tracking:
        accelerator.log(
            {
                f"{prefix}_perplexity": perplexity,
                f"{prefix}_bits_per_token": bits_per_token,
                f"{prefix}_loss": eval_loss,
            },
            step=global_step,
        )

    model.train()
    if not args.disable_neurocache:
        if isinstance(model, NeurocacheModelForCausalLM):
            model.base_cache.reinitialize_cache()
        elif isinstance(model, torch.nn.parallel.DistributedDataParallel):
            model.module.base_cache.reinitialize_cache()
    return eval_loss, bits_per_token, perplexity

def main():
    args = parse_args()

    print(args)

    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.
    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers
    # in the environment
    accelerator_log_kwargs = {}

    if args.with_tracking:
        accelerator_log_kwargs["log_with"] = args.report_to
        accelerator_log_kwargs["project_dir"] = args.output_dir

    accelerator = Accelerator(
        gradient_accumulation_steps=args.gradient_accumulation_steps,
        dispatch_batches=True,
        split_batches=True,
        **accelerator_log_kwargs,
    )

    # Make one log on every process with the configuration for debugging.
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,
    )
    logger.info(accelerator.state, main_process_only=False)
    if accelerator.is_local_main_process:
        transformers.utils.logging.set_verbosity_info()
    else:
        transformers.utils.logging.set_verbosity_error()

    # If passed along, set the training seed now.
    if args.seed is not None:
        set_seed(args.seed)

    # Handle the repository creation
    if accelerator.is_main_process:
        if args.output_dir is not None:
            os.makedirs(args.output_dir, exist_ok=True)
    accelerator.wait_for_everyone()

    # Task configuration
    task_config = dutils.TrainingTaskConfig.from_args(args)

    # Set up seqio tasks
    if args.tokenizer_name is not None:
        dutils.setup_tasks(args.tokenizer_name)
    else:
        dutils.setup_tasks(args.model_name_or_path)

    # Initialize the model
    model = initialize_model(args, accelerator)

    # Optimizer
    # Split weights in two groups, one with weight decay and the other not.
    no_decay = ["bias", "norm.weight"]
    optimizer_grouped_parameters = [
        {
            "params": [
                p
                for n, p in model.named_parameters()
                if (not any(nd in n for nd in no_decay) and p.requires_grad)
            ],
            "weight_decay": args.weight_decay,
        },
        {
            "params": [
                p
                for n, p in model.named_parameters()
                if (any(nd in n for nd in no_decay) and p.requires_grad)
            ],
            "weight_decay": 0.0,
        },
    ]
    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)

    # Scheduler and math around the number of training steps.
    lr_scheduler = get_scheduler(
        name=args.lr_scheduler_type,
        optimizer=optimizer,
        num_warmup_steps=args.num_warmup_steps,
        num_training_steps=args.max_train_steps,
    )

    # Initialize the datasets
    eval_dataloader = initialize_dataloader(
        args.per_device_eval_batch_size, task_config, task_config.eval_split, accelerator
    )

    # Prepare everything with our `accelerator`.
    model, optimizer, lr_scheduler = accelerator.prepare(model, optimizer, lr_scheduler)

    # We need to initialize the trackers we use, and also store our configuration.
    # The trackers initializes automatically on the main process.
    if args.with_tracking:
        experiment_config = vars(args)
        # TensorBoard cannot log Enums, need the raw value
        experiment_config["lr_scheduler_type"] = experiment_config["lr_scheduler_type"].value
        accelerator.init_trackers("clm_no_trainer", experiment_config)

    total_batch_size = (
        args.per_device_train_batch_size
        * accelerator.num_processes
        * args.gradient_accumulation_steps
    )

    logger.info("***** Running training *****")
    logger.info(f"  Instantaneous batch size per device = {args.per_device_train_batch_size}")
    logger.info(
        f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}"
    )
    logger.info(f"  Gradient Accumulation steps = {args.gradient_accumulation_steps}")
    logger.info(f"  Total optimization steps = {args.max_train_steps}")
    # Only show the progress bar once on each machine.
    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)

    # Potentially load in the weights and states from a previous save
    completed_steps, resume_step = 0, 0
    if args.resume_from_checkpoint:
        checkpoint_path = args.resume_from_checkpoint.rstrip("/")

        accelerator.load_state(checkpoint_path)
        accelerator.print(f"Resumed from checkpoint: {checkpoint_path}")

        # need to multiply `gradient_accumulation_steps` to reflect real steps
        training_difference = os.path.splitext(os.path.basename(checkpoint_path))[0]
        resume_step = (
            int(training_difference.replace("step_", "")) * args.gradient_accumulation_steps
        )
        completed_steps = resume_step // args.gradient_accumulation_steps

    if args.save_and_quit:
        if accelerator.is_main_process:
            unwrapped_model = accelerator.unwrap_model(model)
            unwrapped_model.save_pretrained(args.output_dir)
        return

    # update the progress_bar if load from checkpoint
    if not args.only_evaluate:
        train_dataloader = initialize_dataloader(
            args.per_device_train_batch_size,
            task_config,
            task_config.train_split,
            accelerator,
            resume_step=resume_step,
        )

        progress_bar.update(completed_steps)

        model.train()
        logging_loss, logging_steps = 0, 0
        for step, batch in enumerate(train_dataloader):
            with accelerator.accumulate(model):
                target_ids = batch.pop("labels")
                epoch = batch.pop("epochs")[0].item()

                if args.disable_neurocache:
                    batch.pop("start_of_sequence")

                # Forward pass
                outputs = model(**batch)
                loss = F.cross_entropy(
                    outputs.logits.view(-1, outputs.logits.size(-1)).to(torch.float32),
                    target_ids.view(-1),
                    ignore_index=0,  # TODO: get this id from tokenizer
                )

                # Backward pass
                accelerator.backward(loss)

                # Clip gradients
                if accelerator.sync_gradients:
                    accelerator.clip_grad_norm_(model.parameters(), args.max_grad_norm)

                optimizer.step()
                lr_scheduler.step()
                optimizer.zero_grad()

                # Logging
                logging_loss += loss.detach().float()
                logging_steps += 1

            # Checks if the accelerator has performed
            # an optimization step behind the scenes
            if accelerator.sync_gradients:
                progress_bar.update(1)
                completed_steps += 1

                if completed_steps % args.logging_steps == 0:
                    bits_per_token, perplexity = calculate_metrics(logging_loss / logging_steps)
                    # Log metrics with formatting floats
                    progress_bar.set_description(
                        f"step {completed_steps:5d}, "
                        f"train_loss = {logging_loss / logging_steps:.4f}, "
                        f"train_perplexity = {perplexity:.4f}, "
                        f"learning_rate = {optimizer.param_groups[0]['lr']:.4e}"
                    )

                    if args.with_tracking:
                        accelerator.log(
                            {
                                "train_perplexity": perplexity,
                                "train_bits_per_token": bits_per_token,
                                "train_loss": logging_loss / logging_steps,
                                "train_epoch": epoch,
                                "train_lr": optimizer.param_groups[0]["lr"],
                            },
                            step=completed_steps,
                        )

                    logging_steps, logging_loss = 0, 0

                if completed_steps % args.evaluate_every_steps == 0:
                    run_evaluation(
                        args,
                        model,
                        eval_dataloader,
                        accelerator,
                        global_step=completed_steps,
                        prefix="validation",
                    )

                # Save model checkpoint
                if completed_steps % args.checkpointing_steps == 0:
                    output_dir = f"step_{completed_steps}"
                    output_dir = os.path.join(args.output_dir, output_dir)
                    accelerator.save_state(output_dir)

                    # keep only last n checkpoints
                    if args.keep_n_checkpoints is not None:
                        checkpoint_paths = [
                            os.path.join(args.output_dir, path)
                            for path in os.listdir(args.output_dir)
                            if path.startswith("step_")
                        ]
                        checkpoint_paths = sorted(
                            checkpoint_paths, key=lambda path: int(path.split("_")[-1])
                        )
                        for ckpt in checkpoint_paths[: -args.keep_n_checkpoints]:
                            logger.info(f"Removing old checkpoint: {ckpt}")
                            # remove checkpoint
                            shutil.rmtree(ckpt, ignore_errors=True)

                if completed_steps >= args.max_train_steps:
                    break

        if args.with_tracking:
            accelerator.end_training()

        # Save the model
        if args.output_dir is not None:
            accelerator.wait_for_everyone()
            unwrapped_model = accelerator.unwrap_model(model)
            if accelerator.is_main_process:
                if isinstance(unwrapped_model, NeurocacheModelForCausalLM):
                    unwrapped_model.save_pretrained(args.output_dir)
                elif not args.disable_lora:
                    from peft import get_peft_model_state_dict

                    state_dict = get_peft_model_state_dict(model, adapter_name="neurocache")
                    torch.save(state_dict, os.path.join(args.output_dir, "pytorch_model.bin"))
                else:
                    state_dict = unwrapped_model.state_dict()
                    torch.save(state_dict, os.path.join(args.output_dir, "pytorch_model.bin"))
    else:
        run_evaluation(
            args, model, eval_dataloader, accelerator, completed_steps, prefix="validation"
        )

    test_dataloader = initialize_dataloader(
        args.per_device_eval_batch_size, task_config, task_config.test_split, accelerator
    )
    run_evaluation(args, model, test_dataloader, accelerator, completed_steps, prefix="test")

if __name__ == "__main__":
    main()


from pathlib import Path

import pandas as pd
import pytest

from ipumspy import ddi, readers

@pytest.fixture(scope="module")
def cps_ddi(fixtures_path: Path) -> ddi.Codebook:
    return readers.read_ipums_ddi(fixtures_path / "cps_00006.xml")

@pytest.fixture(scope="module")
def cps_df(fixtures_path: Path, cps_ddi: ddi.Codebook) -> pd.DataFrame:
    return readers.read_microdata(cps_ddi, fixtures_path / "cps_00006.csv.gz")

def test_get_variable_info(cps_ddi: ddi.Codebook, cps_df: pd.DataFrame):
    # Does it retrieve the appropriate variable?
    assert cps_ddi.get_variable_info("YEAR").id == "YEAR"

    # Even if the name is not UPPERCASE?
    assert cps_ddi.get_variable_info("year").id == "YEAR"

    # does it give the right description
    assert (
        cps_ddi.get_variable_info("year").description
        == "YEAR reports the year in which the survey was conducted.  YEARP is repeated on person records."
    )

    # does it return the name
    assert cps_ddi.get_variable_info("year").name == "YEAR"

    # codes
    assert cps_ddi.get_variable_info("month").codes == {
        "January": 1,
        "February": 2,
        "March": 3,
        "April": 4,
        "May": 5,
        "June": 6,
        "July": 7,
        "August": 8,
        "September": 9,
        "October": 10,
        "November": 11,
        "December": 12,
    }

    # And does it raise a ValueError if the variable does not exist?
    with pytest.raises(ValueError):
        cps_ddi.get_variable_info("foo")


#!/usr/bin/env python3.10
import argparse
import re
import cairo

def get_args():
    parser = argparse.ArgumentParser(description = "Program to visualize motifs and exons within fasta sequence.")
    parser.add_argument("-f","--fasta", help ="Fasta file to search.", type = str, required = True)
    parser.add_argument("-m","--motif",help = "File containing target motifs.",type = str, required = True)
    return parser.parse_args()
args = get_args()

fasta_file = args.fasta
motif_file = args.motif

class FastaParse:
    def __init__(self, filename):
        self.filename = filename
        self.records = []

    def parse_fasta(self):
        'Parses a fasta file to separate sequence and header lines. Storing them in a list of tuples. Concatenates seq lines into one'
        with open(self.filename) as file:
            record = {'header': '', 'sequence': '', 'exons': []}
            for line in file:
                line = line.strip()
                if line.startswith('>'):
                    if record['header']:
                        self.records.append(record)
                        record = {'header': '', 'sequence': '', 'exons': []}
                    record['header'] = line[1:]
                else:
                    record['sequence'] += line
                    if 'exon' in record['header'].lower():
                        # Extract start and end positions of exon from the header
                        exon_start, exon_end = re.findall(r'\d+', record['header'])
                        # Add exon positions to the list of exons for this record
                        record['exons'].append((int(exon_start), int(exon_end)))
            if record['header']:
                self.records.append(record)
        return self.records

    def header_info(self):
        'Used to extract individual components of fasta headers. Name, start, stop. Can be modified as needed.'
        for record in self.records:
            header = record['header']
            header_info = re.match(r'^(\S+)\s+(\S+):(\d+)-(\d+)', header)
            if header_info:
                name = header_info.group(1)
                start = int(header_info.group(3))
                stop = int(header_info.group(4))

    def find_exons(self):
        'Adds exons to records information , exon is defined by strings of uppercase letters in fasta sequence'
        for record in self.records:
            sequence = record['sequence']
            exons = []
            i = 0
            while i < len(sequence):
                if sequence[i].isupper():
                    start = i
                    while i < len(sequence) and sequence[i].isupper():
                        i += 1
                    end = i
                    exons.append((start, end))
                else:
                    i += 1
            record['exons'] = exons
        return self.records

class Motif:
    def __init__(self, filename):
        self.filename = filename
        self.motifs = []
        self.patterns = []

    def get_motifs(self):
        'Fetches motifs from the motif file, storing in a list, implies that the format is one motif per line in file. '
        with open(self.filename) as f:
            for line in f:
                motif = self.motifs.append(line.strip())
            print(self.motifs)

    def convert_to_regex(self):
        'Converts a list of motifs to their corresponding regex based on nucleic acid notation'
        for motif in self.motifs:
            pattern = ''
            for char in motif:
                #Rules for nucleic acid notation.
                if char in ['a', 'A']:
                    pattern += '[Aa]'
                elif char in ['t', 'T']:
                    pattern += '[Tt]'
                elif char in ['c', 'C']:
                    pattern += '[Cc]'
                elif char in ['g', 'G']:
                    pattern += '[Gg]'
                elif char in ['y', 'Y']:
                    pattern += '[CcTt]'
                elif char in ['r', 'R']:
                    pattern += '[AaGg]'
                elif char in ['w', 'W']:
                    pattern += '[AaTt]'
                elif char in ['s', 'S']:
                    pattern += '[CcGg]'
                elif char in ['k', 'K']:
                    pattern += '[GgTt]'
                elif char in ['m', 'M']:
                    pattern += '[AaCc]'
                elif char in ['d', 'D']:
                    pattern += '[AaGgTt]'
                elif char in ['v', 'V']:
                    pattern += '[AaCcGg]'
                elif char in ['h', 'H']:
                    pattern += '[AaCcTt]'
                elif char in ['n', 'N']:
                    pattern += '[AaCcGgTt]'
                else:
                    pattern += char
            self.patterns.append(pattern)
        print(self.patterns)

    def search_motifs(self, records):
        'Returns a list of tuples for each motif match per record'
        results = []
        for record in records:
            for i, pattern in enumerate(self.patterns):
                matches = re.finditer(pattern, record['sequence'], re.IGNORECASE)
                for match in matches:
                    start = match.start() + 1  # base 1 conversion
                    stop = match.end()
                    #This is the format of the list. Record = header , index is order, start and stop are positions.
                    result = {'record': record, 'pattern_index': i, 'start': start, 'stop': stop}
                    results.append(result)
        return results

class Draw:
    #Define size and spacing for elements, color_map is the colors per motif
    def __init__(self, records, color_map=None):
        self.records = records
        self.width = 800
        self.height = 600
        self.font_size = 16
        self.line_spacing = 20
        self.rect_height = 5
        self.color_map = color_map or []

    def draw_motif_matches(self, results, filename):
        'Takes on positional data for motif and exon matches per record, and draws them to a .png with reference to record lines'
        surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, self.width, self.height)
        ctx = cairo.Context(surface)

        # Sets background to white
        ctx.set_source_rgb(1, 1, 1)
        ctx.paint()

        # Draws record lines
        ctx.set_line_width(2)
        ctx.set_source_rgb(0, 0, 0)
        for i, record in enumerate(self.records):
            y = ((i + 1) * self.height) / (len(self.records) + 2)
            ctx.move_to(0, y)
            ctx.line_to(self.width, y)
            ctx.stroke()

            # Adds header label per record line
            ctx.select_font_face('Arial', cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
            ctx.set_font_size(self.font_size)
            text = record['header']
            x_bearing, y_bearing, width, height = ctx.text_extents(text)[:4]
            ctx.move_to((self.width - width) / 2, y - self.font_size - 2)
            ctx.show_text(text)

            # Draw motifs based on color_map
            ctx.save()
            for result in results:
                if result['record'] == record:
                    pattern_index = result['pattern_index']
                    #Currently defaults extra motifs to black
                    color = self.color_map[pattern_index] if pattern_index < len(self.color_map) else (0, 0, 0)
                    ctx.set_source_rgb(*color)
                    x = (result['start'] - 1) * self.width / len(record['sequence'])
                    y = ((i + 1) * self.height) / (len(self.records) + 2) + self.rect_height
                    width = (result['stop'] - result['start'] + 1) * self.width / len(record['sequence'])
                    ctx.rectangle(x, y, width, self.rect_height)
                    ctx.fill()
            ctx.restore()

            # Draw Exons as Orange
            ctx.save()
            ctx.set_source_rgb(1, 0.5, 0)  # Set the color to orange
            for exon_start, exon_stop in record['exons']:
                exon_width = (exon_stop - exon_start) * self.width / len(record['sequence'])
                exon_x = exon_start * self.width / len(record['sequence'])
                exon_y = y - self.line_spacing - self.font_size - self.rect_height + 33
                ctx.rectangle(exon_x, exon_y, exon_width, self.rect_height)
                ctx.fill()

            ctx.restore()

        # Add legend for color map
        legend_x = self.width * 0.8
        legend_y = self.height * 0.8
        legend_width = self.width * 0.15
        legend_height = self.line_spacing * (len(self.color_map) + 1)
        ctx.rectangle(legend_x, legend_y, legend_width, legend_height)
        ctx.set_source_rgb(0, 0, 0)
        ctx.stroke()

        ctx.select_font_face('Arial', cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
        ctx.set_font_size(self.font_size)
        for i, color in enumerate(self.color_map):
            y = legend_y + (i + 0.5) * self.line_spacing
            ctx.set_source_rgb(*color)
            ctx.rectangle(legend_x + 5, y - self.rect_height / 2, self.rect_height, self.rect_height)
            ctx.fill()
            ctx.move_to(legend_x + self.rect_height + 10, y)
            #Motif 1...2...3.. etc. refers to their order in the initial motif list.
            ctx.show_text(f"Motif {i+1}")

        # Add exon to the legend
        y = legend_y + (len(self.color_map) + 0.5) * self.line_spacing
        ctx.set_source_rgb(1, 0.5, 0)
        ctx.rectangle(legend_x + 5, y - self.rect_height / 2, self.rect_height, self.rect_height)
        ctx.fill()
        ctx.move_to(legend_x + self.rect_height + 10, y)
        ctx.show_text("Exon")

        surface.write_to_png(filename)

parser = FastaParse(fasta_file)
records = parser.parse_fasta()
parser.header_info()
parser.find_exons()
motif = Motif(motif_file)
motif.get_motifs()
motif.convert_to_regex()

matches = motif.search_motifs(records)

#To add more colors, simply add more pixel values to color_map below
color_map = [(1, 0, 0), (0, 1, 0), (0, 0, 1),(1,0.5,1)]
draw = Draw(records,color_map)

#Outputs drawing based on initial fasta file name
draw.draw_motif_matches(matches, f'{fasta_file}.png')


"""from json2xml import json2xml,readfromjson

data = readfromjson("358448 k.kavya 20180323 15-30-00.json")
xml=json2xml.Json2xml(data).to_xml()
print(xml)"""

import os
from json2xml import json2xml,readfromjson
cdir=os.getcwd()
outdir='annotations'
import pdb

for directory in ['data/train','data/test']:
    if not os.path.exists(os.path.join(cdir,directory,outdir)):
        os.mkdir('{}/{}/{}'.format(cdir,directory,outdir))
        #pdb.set_trace()
    for files in os.listdir(os.path.join(cdir,directory,'json')):
        if files.endswith('.json'):
            data=readfromjson(os.path.join(cdir,directory,'json',files))
            xml=json2xml.Json2xml(data).to_xml()
            name=files.replace('.json','.xml')
            txt_file= open(os.path.join(cdir,directory,outdir,name),'w')
            txt_file.write(xml)
            txt_file.close()


import socket
import cv2
import numpy
from multiprocessing import Process
from threading import Thread

class Server:
    def __init__(self):
        # 接收数据量大小
        self.BUFFER_SIZE = 1024
        # 静态socket，用于监听后续发送视频数据的端口
        self.static_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # 0.0.0.0代表监听局域网下所有IPV4地址的连接
        self.host = '0.0.0.0'
        # 静态端口，用于接收后续端口
        self.static_port = 6400
        # 绑定静态socket
        self.static_socket.bind((self.host, self.static_port))
        # 设置监听数
        self.static_socket.listen(1)
        self.port = None
        self.port_list = []

    def StaticHandle(self):
        while True:
            # 等待连接
            client_socket, client_address = self.static_socket.accept()
            # 连接成功后接收后续用于传输数据的端口
            self.ReceivePort(client_socket)

    def handle(self):
        # 建立新的套接字
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # 与后续传输视频的端口绑定
        server_socket.bind((self.host, int(self.port)))
        server_socket.listen(1)
        client_socket, client_address = server_socket.accept()
        self.ReceiveVideo(client_socket, self.port)
        server_socket.close()

    def ReceivePort(self, client_socket):
        # 接收后续传输数据的端口耨
        receive_port = client_socket.recv(self.BUFFER_SIZE)
        self.port = receive_port.decode("utf-8")
        self.port_list.append(self.port)

        # 将接收到的信息发送回客户端进行确认
        return_data = "confirm"
        client_socket.send(return_data.encode('utf-8'))
        handle_thread = Thread(target=self.handle)
        handle_thread.start()
        # self.handle()

    def ReceiveVideo(self, client_socket, name):
        def Receive(sock, count):
            buf = b''
            while count:
                new_buf = sock.recv(count)
                if not new_buf:
                    return None
                buf += new_buf
                count -= len(new_buf)
            return buf

        while True:
            # 首先接收客户端发送的数据长度，16代表接收长度
            try:
                length = Receive(client_socket, 16)
            except socket.error:
                break
            # 若客户端没有向服务端发送数据长度，停止接收
            if length is None:
                self.port_list.remove(name)
                break
            # 接收字符串格式数据
            stringData = Receive(client_socket, int(length))
            # 将字符串格式数据转换为numpy矩阵格式数据
            data = numpy.frombuffer(stringData, numpy.uint8)
            # 对numpy矩阵格式数据进行解码得到图片
            decode_img = cv2.imdecode(data, cv2.IMREAD_COLOR)
            # 使用opencv显示图像，在此处进行修改
            cv2.imshow(f'{name}', decode_img)
            k = cv2.waitKey(30) & 0xff
            if k == 27:
                break
        cv2.destroyAllWindows()

if __name__ == '__main__':
    server = Server()
    t = Thread(target=server.StaticHandle)
    t.start()


import logging
import os
import random

import numpy as np
import torch
import torch.linalg
import yaml
from easydict import EasyDict

def set_seed(s):
    torch.manual_seed(s)
    torch.cuda.manual_seed(s)
    torch.cuda.manual_seed_all(s)
    np.random.seed(s)
    random.seed(s)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

class Logger(object):
    level_relations = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'crit': logging.CRITICAL}

    def __init__(self, path, filename, level='info'):
        if not os.path.exists(path):
            os.makedirs(path)
        self.logger = logging.getLogger(os.path.join(path, filename))
        self.logger.setLevel(self.level_relations.get(level))

        sh = logging.StreamHandler()
        self.logger.addHandler(sh)

        th = logging.FileHandler(os.path.join(path, filename), encoding='utf-8')
        self.logger.addHandler(th)

def inf_iterator(iterable):
    iterator = iterable.__iter__()
    while True:
        try:
            yield iterator.__next__()
        except StopIteration:
            iterator = iterable.__iter__()

class Loader(yaml.SafeLoader):
    """YAML Loader with `!include` constructor."""

    def __init__(self, stream):
        """Initialise Loader."""

        try:
            self._root = os.path.split(stream.name)[0]
        except AttributeError:
            self._root = os.path.curdir

        super().__init__(stream)

def load_config(config_path):
    with open(config_path, 'r') as f:
        config = EasyDict(yaml.load(f, Loader))
    config_name = os.path.basename(config_path)[:os.path.basename(config_path).rfind('.')]
    return config, config_name


"""
Sum of Digits of a Number
You are given a number n. You need to find the sum of digits of n.

Input:
The first line of input contains T denoting the number of testcases. T testcases follow. Each testcase contains one line of input containing n.

Output:
For each testcase, in a newline, print the sum of digits of n.

Your Task:
This is a function problem. You only need to complete the function sumOfDigits that takes n as parameter and returns the sum of digits of n.

Constraints:
1 <= T <= 105
1 <= n <= 107

Examples:
Input:
2
1
99999
Output:
1
45

Explanation:
Testcase 1: Sum of digits for 1 is 1.
Testcase 2: Sum of digits for 99999 is 9 + 9 + 9 + 9 + 9 = 45

** For More Input/Output Examples Use 'Expected Output' option **
"""

def sumOfDigits(n):
    c=0
    for dig in str(n):
        c+=int(dig)
    return(c)

assert sumOfDigits(99999) == 45
assert sumOfDigits(1) == 1

print('The code ran Correctly')


import json

def main():
    with open('demographics.json') as demographics_data:
        counties = json.load(demographics_data)
    print(alphabetically_first_county(counties))
    print(county_most_under_18(counties))
    print(percent_most_under_18(counties))
    print(most_under_18(counties))
    print(state_with_most_counties(counties))

def alphabetically_first_county(counties):
    """Return the county with the name that comes first alphabetically."""
    first = counties[0]["County"]
    for c in counties:
        if c["County"] < first:
            first = c["County"]
    return first

def county_most_under_18(counties):
    """Return the name and state of a county ("<county name>, <state>") with the highest percent of under 18 year olds."""
    highest=0
    county = ""
    for c in counties:
        if c["Age"]["Percent Under 18 Years"] > highest:
            highest = c["Age"]["Percent Under 18 Years"]
            county = c["County"] + ", " + c["State"]
    return county

def percent_most_under_18(counties):
    """Return the highest percent of under 18 year olds."""
    highest=0
    for c in counties:
        if c["Age"]["Percent Under 18 Years"] > highest:
            highest = c["Age"]["Percent Under 18 Years"]
    return highest

def most_under_18(counties):
    """Return a list with the name and state of a county ("<county name>, <state>") and the percent of under 18 year olds for a county with the highest percent of under 18 year olds."""
    result = ["", 0]
    for c in counties:
        if c["Age"]["Percent Under 18 Years"] > result[1]:
            result[1] = c["Age"]["Percent Under 18 Years"]
            result[0] = c["County"] + ", " + c["State"]
    return result

def state_with_most_counties(counties):
    """Return a state that has the most counties."""
    #Make a dictionary that has a key for each state and the values keep track of the number of counties in each state
    states={}
    for c in counties:
        if c["State"] in states:
            states[c["State"]] += 1
        else:
            states[c["State"]] = 1
    #Find the state in the dictionary with the most counties
    highest = 0
    state = ""
    for s in states:
        if states[s] > highest:
            highest = states[s]
            state = s
    #Return the state with the most counties
    return state

def your_interesting_demographic_function(counties):
    """Compute and return an interesting fact using the demographic data about the counties in the US."""

if __name__ == '__main__':
    main()


class Solution:
    """
    Implementation of https://leetcode.com/problems/house-robber
    """
    def __init__(self):
        self.memo={}

    def rob(self, nums: List[int]) -> int:
        return self.robr(nums,0)

    def robr(self, nums: List[int], i: int) -> int:
        if len(nums[i:]) == 0:
            r =0
            self.memo[i]=r
            return r
        elif len(nums[i:]) == 1:
            r =nums[i]
            self.memo[i]=r
            return r
        elif len(nums[i:]) == 2:
            r =max(nums[i],nums[i+1])
            self.memo[i]=r
            return r

        if i+2 in self.memo:
            mi2 = self.memo[i+2]
        else:
            mi2 = self.robr(nums,i+2)
            self.memo[i+2] = mi2

        if i+1 in self.memo:
            mi1 = self.memo[i+1]
        else:
            mi1 = self.robr(nums,i+1)
            self.memo[i+1] = mi1
        return max(nums[i] + mi2, mi1)


## Logistic Regression Model - HOMEMADE
import theano.tensor as T
from pylearn2.costs.cost import Cost, DefaultDataSpecsMixin
from pylearn2.models.model import Model
from pylearn2.utils import sharedX
from pylearn2.space import VectorSpace
import numpy as np

## The order of DefaultDataSpecsMixin and Cost matters!
class LogisticRegressionCost(DefaultDataSpecsMixin, Cost):
    supervised = True ## specify supervised learning costs

    ## need model to map input to output, need data to test with target
    def expr(self, model, data, **kwargs):
        space, source = self.get_data_specs(model) ## model's data specification
        space.validate(data) ## use model's vector space to validate data

        ## All X, y, yhat are theano variables
        X, y = data ## since it is supervised cost, we got both
        yhat = model.logistic_regression(X) ## call model to map X to yhat
        loss = -(y * T.log(yhat)).sum(axis = 1) ## rowwise selection
        return loss.mean() ## take the mean

class LogisticRegression(Model):
    def __init__(self, nvis, nclasses):
        super(LogisticRegression, self).__init__() ## call superclass Model constructure

        self.nvis = nvis ## standard name for ninputs
        self.nclasses = nclasses ## standard name for noutputs

        ## all model parameters are shared variable in Theano
        ## they usually come with an initialization and a name
        W_value = np.random.uniform(size = (self.nvis, self.nclasses))
        self.W = sharedX(W_value, "W")
        b_value = np.zeros(self.nclasses)
        self.b = sharedX(b_value, "b")
        ## all model parameters should be recorded in self._params
        ## for gradient calculation and house-keeping
        self._params = [self.W, self.b]

        ## construct the data specification
        self.input_space = VectorSpace(dim = self.nvis)
        self.output_space = VectorSpace(dim = self.nclasses)

    def logistic_regression(self, X):
        return T.nnet.softmax(T.dot(X, self.W) + self.b)


import pandas as pd

classes = pd.read_csv("classes.csv")

excerpts = pd.read_csv("labeled_excerpts.csv")

classes["frequency"] = excerpts.groupby("class_id")["excerpt"].count()

# print(classes)
# print(classes["frequency"].sort_values(reverse=True))

classes.to_csv("classes_with_frequency.csv")


# Problem statement:
# User inputs daily temperatures,
# we need to calculate average temperature and number of days that have crossed the average temperature.

# Solution without using array/lists
from numpy import average

numDays = int(input("How many day's temperature? "))
total = 0
for i in range(1, numDays+1):
    dayTemp = int(input("Day "+str(i)+"'s high temperature: "))
    total += dayTemp
average = round(total/numDays, 2)
print("Average: ", average)

# With above approach we can get avg but can't count #days with > avg cuz that needs a second pass through data.
# which we did not maintain. Taking input once more for this would be a bad approach.
# So, here comes array/list into the picture.

numDays = int(input("How many day's temperature? "))
total = 0
temp = []
for i in range(numDays):
    dayTemp = int(input("Day "+str(i+1)+"'s high temperature: "))
    temp.append(dayTemp)
    total += temp[i]
average = round(total/numDays, 2)
print("Average: ", average)
above = 0
for i in temp:
    if i > average:
        above += 1
print(str(above) + " day(s) above average")


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Perform performance testing of aba-web
"""
from os import listdir, getpid
from os.path import isfile, join, splitext
import logging
from aba.aba_parser import ABA_Parser
from aba.aba_perf_logger import ABA_Perf_Logger

## Safety measure so that the process won't overshoot my RAM limit
import sys
if sys.platform == "win32":
    import perf_memory_limiter
    five_gb = 5 * 1024 * 1024 * 1024
    perf_memory_limiter.set_limit(getpid(), five_gb)

logging.basicConfig(filename=join('logs', 'PerfTests.log'),level=logging.INFO,format='[%(asctime)s] %(levelname)s:%(name)s: %(message)s',datefmt='%Y-%m-%d %H:%M:%S') # Note: making debug level DEBUG will slow down the process A LOT due to disk I/O

logging.info("Start performance test")
global_perf_logger = ABA_Perf_Logger("Overall performance test")
global_perf_logger.start()

runonly = []
with open("perf_test_runonly.txt", "r") as f:
    runonly = f.readlines()
runonly = [x.strip() for x in runonly]

path = "perf_test_data"
test_files = [f for f in listdir(path) if splitext(f)[0] in runonly and isfile(join(path, f)) and splitext(f)[1] == ".txt"]

for file in test_files:
    with open(join(path, file), 'r') as f:
        source_code = f.read()
        logging.info("Start running %s", file)
        perf_logger = ABA_Perf_Logger("%s" % file)
        perf_logger.start()

        parser = ABA_Parser(source_code)
        parse_errors = parser.parse()

        if len(parse_errors) > 0:
            logging.info("Parse error: %s", ', '.join(parse_errors))
        else:
            try:
                parser.construct_aba()
            except MemoryError:
                logging.error("Out of memory!")
            except Exception as exp: # "trick" to continue to next test even when exception occurs during runtime
                logging.error("Runtime error: %s", str(exp))
            except BaseException as exp: # catch those that can't be caught
                logging.error("Unexpected error: %s", str(exp))

        perf_logger.end()

global_perf_logger.end()


# -*- coding: utf-8 -*-
import scrapy

class KotaindustriSpider(scrapy.Spider):
    name = 'kotaindustri'
    allowed_domains = ['www.kotaindustri.co.id']
    start_urls = ['https://www.kotaindustri.co.id/all-categories/']

    def parse(self, response):
        for node in response.css('#directorist a'):
            node_url = node.css('a::attr(href)').get().strip()
            yield response.follow(node_url, callback=self.parse_category)

    def parse_category(self, response):
        for node in response.css('.atbdp_column .atbd_listing_title > a'):
            node_url = node.css('a::attr(href)').get().strip()
            yield response.follow(node_url, callback=self.parse_detail)

    def parse_detail(self, response):
        contact = response.css('.atbd_contact_information_module')
        if contact is None:
            return
        category = response.css('.directory_tag span a::text').get() or ''
        name = response.css('h1::text').get() or ''
        address = ''
        city = ''
        phone = ''
        email = ''
        website = ''
        description = []
        url = response.url or ''

        # description
        for txt in response.css('.about_detail p::text'):
            description.append(txt.get().strip())
        description = '. '.join(description)
        description = description.replace('..', '.')
        description = description.replace('. . ', '. ')

        # table information
        for tr in response.css('.atbd_contact_info > ul > li'):
            k = tr.css('.atbd_info_title::text').get()
            v = tr.css('.atbd_info::text').get()
            if k is None or len(k) == 0:
                continue
            elif 'Alamat' in k:
                address = v
            elif 'Telepon' in k:
                phone = tr.css('.atbd_info a::text').get()
            elif 'Email' in k:
                email = tr.css('.atbd_info a::text').get()
            elif 'Website' in k:
                website = v

        if len(email) == 0:
            self.logger.info('{} : EMPTY EMAIL'.format(url))
        if len(phone) == 0:
            self.logger.info('{} : EMPTY PHONE'.format(url))

        if len(email) > 0 and len(phone) > 0:
            yield {
                'category': category.strip(),
                'name': name.strip(),
                'address': address.strip(),
                'city': city.strip(),
                'phone': phone.strip(),
                'email': email.strip(),
                'website': website.strip(),
                'description': description.strip(),
                'url': url.strip(),
            }


# import
import urllib.request

def downlaod(category):
    webx_loc = 'http://celestrak.com/NORAD/elements/' + category + '.txt'
    disk_loc = 'TLE/' + category + '.txt'
    urllib.request.urlretrieve(webx_loc, disk_loc)

list_category = [

"tle-new", "stations", "visual",
"1999-025", "iridium-33-debris", "cosmos-2251-debris",
"2012-044", "weather", "noaa",
"goes", "resource", "sarsat",
"dmc", "tdrss", "argos",
"geo", "intelsat", "ses",
"iridium", "iridium-NEXT", "orbcomm",
"globalstar", "amateur", "x-comm",
"other-comm", "gorizont", "raduga",
"molniya", "gps-ops", "glo-ops",
"galileo", "beidou", "sbas",
"nnss", "musson", "science",
"geodetic", "engineering", "education",
"military", "radar", "cubesat",
"other"

]

for element in list_category:
    downlaod(element)


import logging
import time

from selenium.common.exceptions import ElementNotVisibleException
from selenium.webdriver.common.by import By

from page_objects.base import BaseElement
from page_objects.base import BaseLoadingElement
from page_objects.base import BasePage
from page_objects.base import TextInput

class Page(BasePage):

    def __init__(self, driver):
        super().__init__(driver=driver, desc='Pokedex Page', url='https://www.pokemon.com/us/pokedex/')
        self._locators = dict()
        self._locators['main_nav'] = (By.CSS_SELECTOR, 'nav.main')

        self._locators['execute_search_button'] = (By.CSS_SELECTOR, 'input.button-search')
        self._locators['loading_indicator'] = (By.CSS_SELECTOR, 'div.loader')
        self._locators['no_results'] = (By.CSS_SELECTOR, 'div.no-results')
        self._locators['search_field'] = (By.CSS_SELECTOR, '#searchInput')
        self._locators['search_result'] = (By.CSS_SELECTOR, 'li.animating')
        self._locators['sort_dropdown'] = (By.CSS_SELECTOR, 'section.overflow-visible > div > div > div.custom-select-menu')

        self._locators['load_more_button'] = (By.CSS_SELECTOR, '#loadMore > span')

        self._locators['footer'] = (By.CSS_SELECTOR, 'div.footer-divider')
        return

    # Basic Filters

    def click_execute_search_button(self):
        element = self.driver.find_element(*self._locators['execute_search_button'])
        logging.info("Clicking execute search button.")
        element.click()
        return

    def find_search_field_text_input_object(self):
        element = self.driver.find_element(*self._locators['search_field'])
        return TextInput(element)

    def find_sort_dropdown_object(self):
        element = self.driver.find_element(*self._locators['sort_dropdown'])
        return SortDropdown(element=element)

    # Search Results

    def all_search_results_names_displayed(self):
        results = self.find_search_result_objects()
        return [i.name for i in results]

    def all_search_results_numbers_displayed(self):
        results = self.find_search_result_objects()
        return [i.number for i in results]

    def no_results_found(self):
        return self.driver.find_element(*self._locators['no_results']).is_displayed()

    @property
    def number_of_results(self):
        return len(self.find_search_result_objects())

    def find_search_result_objects(self):
        elements = self.driver.find_elements(*self._locators['search_result'])
        return [SearchResult(i) for i in elements]

    # Load More Button

    def click_load_more_button(self):
        self._verify_load_more_button_is_displayed()
        self.scroll_to_load_more_button()
        element = self.driver.find_element(*self._locators['load_more_button'])
        logging.info("Clicking 'Load More' button.")
        element.click()
        return

    def load_more_button_is_displayed(self):
        elements = self.driver.find_elements(*self._locators['load_more_button'])
        if len(elements) == 0:
            return False
        else:
            return elements[0].is_displayed()

    def scroll_to_load_more_button(self):
        self._verify_load_more_button_is_displayed()
        load_more_button_element = self.driver.find_element(*self._locators['load_more_button'])

        # This method scrolls and returns coordinates.
        load_more_button_location = load_more_button_element.location_once_scrolled_into_view

        # But this method alone isn't good enough. The nav could still cover the button, resulting in an
        #   ElementClickInterceptedException.

        nav_element = self.driver.find_element(*self._locators['main_nav'])
        load_more_button_y = load_more_button_location['y']
        if load_more_button_y < nav_element.size['height']:
            scroll_distance = -(nav_element.size['height'] - load_more_button_y)
            self.driver.execute_script("window.scrollBy(0,{})".format(scroll_distance))
        return

    def _verify_load_more_button_is_displayed(self):
        if not self.load_more_button_is_displayed():
            log_str = "'Load More' button is not displayed."
            logging.error(log_str)
            raise ElementNotVisibleException(log_str)
        return

    # Misc

    def accept_cookies(self):
        cookie_modal = CookieModal(driver=self.driver)

        if not cookie_modal.is_loaded():
            logging.debug(f"{cookie_modal.desc} is not displayed. No action needed.")
            return

        cookie_modal.click_ok_button()
        cookie_modal.wait_until_closed()
        return

    def loading_indicator_is_displayed(self):
        elements = self.driver.find_elements(*self._locators['loading_indicator'])
        if len(elements) == 0:
            return False
        else:
            return elements[0].is_displayed()

    def is_loaded(self):
        return not self.loading_indicator_is_displayed()

    def scroll_to_footer(self):
        element = self.driver.find_element(*self._locators['footer'])
        element.location_once_scrolled_into_view
        return

class SortDropdown(BaseElement):

    def __init__(self, element):
        super().__init__(element=element, desc='Sort Dropdown')
        self._locators = dict()
        self._locators['current_option'] = (By.CSS_SELECTOR, 'label')
        self._locators['option'] = (By.CSS_SELECTOR, 'li')
        return

    # Displaying Options

    def click_dropdown(self):
        logging.info(f"Clicking {self.desc}.")
        self.element.click()
        time.sleep(0.1)
        return

    def display_options(self):
        if self.options_are_displayed():
            logging.debug('Options are already displayed. No action needed.')
            return
        self.click_dropdown()
        return

    def options_are_displayed(self):
        label_element = self.element.find_element(*self._locators['current_option'])
        return 'opened' in label_element.get_attribute('class')

    def _verify_options_are_displayed(self):
        if not self.options_are_displayed():
            log_str = "Options are not displayed."
            logging.error(log_str)
            raise ElementNotVisibleException(log_str)
        return

    # Setting/Getting Options

    def click_option(self, option):
        self._verify_options_are_displayed()
        self._verify_option_exists(option)

        element = self._find_option_element(option=option)
        logging.info(f"Clicking sort option '{option}'.")
        element.click()
        return

    def _find_option_element(self, option):
        self._verify_options_are_displayed()

        elements = self.element.find_elements(*self._locators['option'])
        for i in elements:
            if option.lower() == i.text.lower():
                return i

        log_str = f"Invalid option '{option}' specified."
        logging.error(log_str)
        raise ValueError(log_str)

    @property
    def selected_option(self):
        return self.element.find_element(*self._locators['current_option']).text

    @selected_option.setter
    def selected_option(self, option):
        self.display_options()
        self._verify_option_exists(option=option)
        self.click_option(option=option)
        return

    def _verify_option_exists(self, option):
        self._find_option_element(option=option)
        return

class SearchResult(BaseElement):

    def __init__(self, element):
        super().__init__(element=element, desc='Search Result')

        self._locators = dict()
        self._locators['name'] = (By.CSS_SELECTOR, 'h5')
        self._locators['number'] = (By.CSS_SELECTOR, 'p.id')

        self.desc = f"Search Result - {self.number} {self.name}"
        return

    @property
    def name(self):
        return self.element.find_element(*self._locators['name']).text

    @property
    def number(self):
        return self.element.find_element(*self._locators['number']).text

    def __str__(self):
        return self.desc

class CookieModal(BaseLoadingElement):

    def __init__(self, driver):
        super().__init__(driver=driver, desc="Cookie Modal")
        self._locators = dict()
        self._locators['cookie_modal'] = (By.CSS_SELECTOR, 'div.ot-sdk-container')
        self._locators['ok_button'] = (By.CSS_SELECTOR, '#onetrust-accept-btn-handler')
        return

    def click_ok_button(self):
        button = self.driver.find_element(*self._locators['ok_button'])
        logging.info(f"Clicking 'OK' on {self.desc}")
        button.click()
        return

    def is_loaded(self):
        elements = self.driver.find_elements(*self._locators['cookie_modal'])
        if len(elements) == 0:
            return False
        else:
            return elements[0].is_displayed()


import numpy as np
from mss import mss

def capture_screenshot_as_numpy():
    with mss() as sct:
        # Define the region for capturing screenshot. Adjust the left and top accordingly.
        monitor = {"top": 0, "left": 0, "width": 500, "height": 500}

        # Capture screenshot
        screenshot = sct.grab(monitor)

        # Convert the screenshot's raw bytes to a numpy array
        np_screenshot = np.array(screenshot)

        return np_screenshot

# Capture the screenshot and save as numpy array
np_image = capture_screenshot_as_numpy()

print(np_image)  # This


#!/bin/python3

'''
Problem - https://www.hackerrank.com/challenges/
staircase/problem?h_r=next-challenge&h_v=zen
'''

def staircase(n):
    for i in range(0, n):
        a = [' '] * (n - i - 1)
        b = ['#'] * (i + 1)
        c = a + b
        d = ''.join(c)
        print(d)

if __name__ == '__main__':
    n = int(input())

    staircase(n)


import cv2
import numpy as np
import joblib

# 最开始计算轮廓会出现异常现象，会出现count =1 的错误
def count_conts(frame, contours, area ):
    count = 0
    for contour in contours:
        if cv2.contourArea(contour) < area:
            continue
        count += 1
        (x, y, w, h) = cv2.boundingRect(contour)
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 2)
    return count

""" 获取轮廓面积大于预定值的轮廓，用于追踪目标"""
def get_contours(contours,area):
    conts = []
    for contour in contours:
        if cv2.contourArea(contour) < area:
            continue
        conts.append(contour)
    return conts

# def get_area_center(points):
#
#     return (x,y)

"""
    计算区域中心，左中右进行排序
"""
def get_center(pts):
    x = 0
    y = 0

    for i in range(len(pts)):
        x += pts[i][0]
        y += pts[i][1]
    x /= len(pts)
    y /= len(pts)
    x_left = (pts[0][0]+pts[1][0])/2
    y_left = (pts[0][1]+pts[1][1])/2
    x_right = (pts[2][0] + pts[3][0]) / 2
    y_right = (pts[2][1] + pts[3][1]) / 2

    return (int(x),int(y)),(int(x_left),int(y_left)),(int(x_right),int(y_right))

"""
    异常分类：
        单目标异常：
            异常一：穿过区域,代码表示，3s内穿过中心点
            异常二：在异常区停留3s，代码表示3s内未穿过中心点
        todo：多目标异常
"""
if __name__ == '__main__':
    cap = cv2.VideoCapture(1)

    """
        获取手工标记的roi区域
    """
    fgbg = cv2.createBackgroundSubtractorMOG2()
    Roi = joblib.load("config1.pkl")
    pts = Roi.get('ROI1')
    points = np.array(pts, np.int32)
    points = points.reshape((-1, 1, 2))

    """
        获取手工标记的异常区域
    """
    Roi_error = joblib.load("config2.pkl")
    pts_error = Roi_error.get('ROI2')
    points_error = np.array(pts_error, np.int32)
    points_error = points_error.reshape((-1, 1, 2))

    """
        获取异常区域中
    """
    center_point,left_point,right_point = get_center(pts_error)

    print(center_point, left_point, right_point)

    """
        time_count: 解决第一次获取轮廓会获取最外层轮廓的异常现象
    """
    time_count = 0

    """
        用于判断异常类型
    """
    frequnence = cv2.getTickFrequency()
    start_time = 0
    flag = 0  # 用于计时，需要保证前后两人进入监控区的时间误差为1s
    error_time_first = 0  # 用于第一次异常判断，判断属于一个物体异常还是两个物体异常
    flag_error_first = 0
    flag_error_second = 0 #用于第二次异常判断
    error_type = 0 # 用于判断异常类型

    while (cap.isOpened):
        success, frame = cap.read()
        if success:
            """ 标记roi区域，为避免影响roi区域以外的区域不进行训练"""
            mask = np.zeros(frame.shape, np.uint8)
            # mask = cv2.polylines(mask, [points], True, (255, 255, 255), 2)
            mask2 = cv2.fillPoly(mask.copy(), [points], (255, 255, 255))  # 用于求 ROI
            mask3 = cv2.fillPoly(mask.copy(), [points], (0, 255, 0))  # 用于 显示在桌面的图像
            # show_image = cv2.addWeighted(src1=frame, alpha=0.8, src2=mask3, beta=0.2, gamma=0)
            ROI = cv2.bitwise_and(mask2, frame)  #  获取roi区域掩模
            # cv2.imshow("roi",ROI)
            fgmask_roi = fgbg.apply(ROI)  #  背景减除算法应用
            ret, fgmask_roi = cv2.threshold(fgmask_roi, 254, 255, cv2.THRESH_BINARY) #  获取无阴影的图片
            # kernel = np.ones((3,1),np.uint8)
            # fgmask_roi = cv2.erode(fgmask_roi,kernel) #  腐蚀算法
            frame = cv2.polylines(frame, [points], True, (255, 0, 0), 2)  # 标定监控区域
            frame = cv2.polylines(frame, [points_error], True, (0, 255, 0), 2)
            frame = cv2.circle(frame, center_point, 3, (0, 0, 255), -1)
            frame = cv2.circle(frame, left_point, 3, (0, 0, 255), -1)
            frame = cv2.circle(frame, right_point, 3, (0, 0, 255), -1)

            contours, _ = cv2.findContours(fgmask_roi, cv2.RETR_EXTERNAL,
                                                   cv2.CHAIN_APPROX_SIMPLE)
            """
                直接跳过第一帧的画面从第二帧开始算起
            """
            if time_count == 0:
                time_count += 1
                continue
            count = count_conts(frame,contours,7000)

            """ 当检测区域中出现第一个异常开始计时判断异常"""
            if count == 1 and flag == 0 :
                start_time = cv2.getTickCount()
                error_time_first = start_time
                flag = 1
                # print(start_time)

            """ 保证err_time_first在start_time计时之后开始计时"""
            if flag == 1:
                error_time_first = cv2.getTickCount()
                if (error_time_first-start_time)/frequnence > 0.7 and flag_error_first == 0:
                    if count == 1:
                        print("属于第一大类异常")
                        flag_error_first = 1
                    else:
                        print("属于第二大类异常")
                        flag_error_first = 2
                    flag =2

            """若异常为第一大异常"""
            if flag_error_first == 1:
                # print("判断第一大类异常")
                conts = get_contours(contours,7000)
                # print(conts[0])
                if conts:   # 如果轮廓不为空

                    dist = cv2.pointPolygonTest(conts[0],center_point,False)
                    # print(dist)
                    if dist == 1.0 and flag_error_second == 0:
                        start_time = cv2.getTickCount()
                        # print("error1 starttime= ", start_time)
                        error_time_first = start_time
                        flag_error_second = 1
                    if flag_error_second == 1:

                        dist1 = cv2.pointPolygonTest(conts[0], left_point, False)
                        error_time_first = cv2.getTickCount()
                        if dist1 == 1.0 and dist == -1:
                            print("发生异常1")
                            error_type = 1
                            flag_error_first = 3

                        elif (error_time_first-start_time)/frequnence > 1:

                            dist2 = cv2.pointPolygonTest(conts[0],right_point,False)
                            # if dist == -1:
                            #     print("发生异常1")
                            #     error_type =1
                            #
                            # else:
                            #     error_type = 2
                            #     print("发生异常2")

                            if dist2 == 1:
                                print("发生异常3")
                                error_type = 3
                                flag_error_first = 3
                            elif (error_time_first-start_time)/frequnence > 3:
                                print("发生异常2")
                                error_type = 2
                                flag_error_first = 3

            elif flag_error_first==2:
                """若异常为第二大类异常"""
                # print("判断第一大类异常")
                conts = get_contours(contours, 7000)
                # print(conts[0])
                if conts:
                    dist = cv2.pointPolygonTest(conts[0], center_point, False)

                    if dist == 1 and flag_error_second==0:
                        start_time = cv2.getTickCount()
                        error_time_first = start_time
                        flag_error_second = 1
                    if flag_error_second == 1:
                        error_time_first = cv2.getTickCount()
                        if (error_time_first-start_time)/frequnence > 2:
                            if len(conts)>1:
                                print("发生异常4")
                                error_type =4

                            else:
                                error_type = 5
                                print("发生异常5")
                            flag_error_first = 3
                    # print("判断第二大类异常")

            if error_type == 1:
                frame = cv2.putText(frame, 'Error Happening, Error Type: 1', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
                                    (0, 0, 255), 2)
            elif error_type ==2:
                frame = cv2.putText(frame, 'Error Happening, Error Type: 2', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
                                    (0, 0, 255), 2)
            elif error_type == 3:
                frame = cv2.putText(frame, 'Error Happening, Error Type: 3', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
                                    (0, 0, 255), 2)
            elif error_type == 4:
                frame = cv2.putText(frame, 'Error Happening, Error Type: 4', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
                                    (0, 0, 255), 2)
            elif error_type == 5:
                frame = cv2.putText(frame, 'Error Happening, Error Type: 5', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
                                    (0, 0, 255), 2)

            cv2.namedWindow("src",cv2.WINDOW_NORMAL)
            cv2.resizeWindow("src",800,600)
            cv2.imshow("src", frame)
            cv2.namedWindow("mask", cv2.WINDOW_KEEPRATIO)
            cv2.resizeWindow("mask", 800, 600)
            cv2.imshow("mask", fgmask_roi)
            key = cv2.waitKey(1) & 0xFF
            '''重置'''
            if key == ord('r'):
                print("重置实验")
                start_time = 0
                flag = 0  # 用于计时，需要保证前后两人进入监控区的时间误差为1s
                error_time_first = 0  # 用于第一次异常判断，判断属于一个物体异常还是两个物体异常
                flag_error_first = 0
                flag_error_second = 0  # 用于第二次异常判断
                error_type = 0  # 用于判断异常类型

            if key == ord('q'):
                break
        else:
            break
cap.release()
cv2.destroyAllWindows()


# %% Load and manipulate the raw dataframe
import geopandas
import pandas
from pyproj import Transformer

def get_iso_centre(row, dim):
    if row['iso_type'] == 'from_centroid':
        return row[f'centre_{dim}']
    elif row['iso_type'] == 'from_nearest_node':
        return row[f'node_{dim}']
    elif row['iso_type'] == 'from_nearest_stop':
        return row[f'stop_{dim}']
    else:
        return row[f'centre_{dim}']

df = geopandas.read_file('UK_Travel_Area_Isochrones_(Nov_Dec_2022)_by_Public_Transport_and_Walking_for_Northern_Ireland_-_Generalised_to_10m.geojson')

trans = Transformer.from_crs(
    "EPSG:29902",
    "EPSG:4326",
    always_xy=True,
)
for pre in ['centre', 'node', 'stop']:
    df[f'{pre}_X'], df[f'{pre}_Y'] = trans.transform(df[f'{pre}_X'].values, df[f'{pre}_Y'].values)

df['Travel Minutes'] = pandas.to_numeric(df['iso_cutoff'] / 60, downcast='integer')

df['iso_centre_X'] = df.apply(get_iso_centre, axis=1, dim='X')
df['iso_centre_Y'] = df.apply(get_iso_centre, axis=1, dim='Y')

# %%
df.head()

# %% Reduce down the size of the dataframe and then split by Small Area to minimise data accesses from the app
def get_difference(a, b):
    if a.geometry.isna().all() or b.geometry.isna().all():
        return a
    return a.overlay(b, how='difference')

reduced = df[['SA2011','iso_type','Travel Minutes','iso_centre_X','iso_centre_Y', 'geometry']]
for sa in reduced['SA2011'].unique():
    original = reduced[reduced['SA2011']==sa]
    min60 = get_difference(original.loc[lambda df: df['Travel Minutes'] == 60, :], original.loc[lambda df: df['Travel Minutes'] == 45, :])
    min45 = get_difference(original.loc[lambda df: df['Travel Minutes'] == 45, :], original.loc[lambda df: df['Travel Minutes'] == 30, :])
    min30 = get_difference(original.loc[lambda df: df['Travel Minutes'] == 30, :], original.loc[lambda df: df['Travel Minutes'] == 15, :])
    min15 = original.loc[lambda df: df['Travel Minutes'] == 15, :]
    pandas.concat([min60, min45, min30, min15]).to_file(f'{sa}.geojson', driver='GeoJSON')

# %%
df.groupby('iso_type').count()

# %%
original = reduced[reduced['SA2011']=='N00000897']
min60 = original.loc[lambda df: df['Travel Minutes'] == 60, :].overlay(original.loc[lambda df: df['Travel Minutes'] == 45, :], how='difference')
min45 = original.loc[lambda df: df['Travel Minutes'] == 45, :].overlay(original.loc[lambda df: df['Travel Minutes'] == 30, :], how='difference')
min30 = original.loc[lambda df: df['Travel Minutes'] == 30, :].overlay(original.loc[lambda df: df['Travel Minutes'] == 15, :], how='difference')
min15 = original.loc[lambda df: df['Travel Minutes'] == 15, :]
pandas.concat([min60, min45, min30, min15]).to_file('N00000897.geojson', driver='GeoJSON')

# %%


import pymysql
import schedule
import time

estacion=1

class DataBase:
    def __init__(self):
        self.connection = pymysql.connect(
            host='localhost',
            user='root',
            password='',
            db='proyecto_grado'
        )

        self.cursor = self.connection.cursor()

        print("Conexión establecida exitosamente")

    def select_user(self, id):
        sql = 'SELECT * FROM usuarios WHERE id={}'.format(id)

        try:
            self.cursor.execute(sql)
            user = self.cursor.fetchone()
            print("Id: ", user[0])
            print("Nombre: ", user[1])
            print("Apellidos: ", user[2])

        except Exception as e:
            raise

    def insert_humedadrelativa(self):
        datos = 'datos.txt'
        with open(datos) as f_obj:
            lines = f_obj.readlines()

        sql = "INSERT INTO humedad_relativa (estacion, valor) VALUES ('{}', '{}')".format(estacion, lines[0])
        sql2 = "INSERT INTO temperatura (estacion, valor) VALUES ('{}', '{}')".format(estacion, lines[1])
        sql3 = "INSERT INTO presion_atmosferica (estacion, valor) VALUES ('{}', '{}')".format(estacion, lines[2])
        sql4 = "INSERT INTO pmtwo (estacion, valor) VALUES ('{}', '{}')".format(estacion, lines[3])
        sql5 = "INSERT INTO pmten (estacion, valor) VALUES ('{}', '{}')".format(estacion, lines[4])

        try:
            self.cursor.execute(sql)
            self.cursor.execute(sql2)
            self.cursor.execute(sql3)
            self.cursor.execute(sql4)
            self.cursor.execute(sql5)
            self.connection.commit()

            print("Insercción de datos exitosa! ")
        except Exception as e:
            raise

def calldb():
    database = DataBase()
    database.insert_humedadrelativa()

schedule.every(10).seconds.do(calldb)

while 1:
    schedule.run_pending()
    time.sleep


import requests

BASE = "http://127.0.0.1:5000/"

data = [{"likes":80, "name": "Camilo", "views": 10000},
{"likes":12, "name": "How to make videos", "views": 123123},
{"likes":100, "name": "How to make a rest api in python", "views": 22}]

for i in range(len(data)):
    response = requests.put(BASE + "video/" + str(i),  data[i])
    print(response.json())

# response = requests.put(BASE + "video/1", {"likes":10, "name": "Camilo", "views": 10000})
# print(response.json())
# input()

input()
response - requests.delete(BASE + "video/0")
print(response.json())

input()

response = requests.get(BASE + "video/1")
print(response.json())


from sklearn.datasets import load_iris
from sklearn.utils import shuffle
import numpy as np
import tqdm
import matplotlib.pyplot as plt
### multi classification with decision tree

class Binary_inquiry_train:
    def __init__(self,X,Y): # input unnomarlized X and divided label into 0 and 1
        self.X = self.normalize(X)
        self.Y = Y
        self.W = np.random.rand(self.X.shape[1], 1)
        self.bias = np.random.rand()
        self.learning_rate = 0.01
        self.hypothesis = self.hypothesis_(self.X, self.W, self.bias)

        for i in tqdm.tqdm(range(8000)):
            self.W, self.bias, self.acc = self.GD_optimizer(self.X, self.Y, self.W, self.hypothesis, self.bias, self.learning_rate)
            self.hypothesis = self.sigmoid(np.dot(self.X, self.W) + self.bias).reshape((self.X.shape[0],))
            self.H = self.Binary_cross_entropy(self.hypothesis, self.Y)

        print("H = {}, train_acc = {}".format(self.H, self.acc))
        self.output = (self.W, self.bias)

    def normalize(self,X):
        mean = np.mean(X, axis=0, dtype=np.float32)
        std = np.std(X, axis=0, dtype=np.float32)
        X = (X - mean) / std
        return X

    def sigmoid(self,X):
        return 1/(1 + np.exp(-X))

    def Round_H_Round_W(self,X, hypothesis, Y):
        return ( np.dot(X.T,( hypothesis - Y )) / X.shape[0] ).reshape(4,1)

    def Round_H_Round_bias(self,X, hypothesis, Y):
        return np.sum( hypothesis - Y ) / X.shape[0]

    def Binary_cross_entropy(self,hypothesis, Y):
        H = - np.sum(( ( hypothesis - Y ) * np.log(hypothesis) + ( 1 - Y ) * np.log(1 - hypothesis)  ))
        return H

    def hypothesis_(self,X,W,bias):
        return self.sigmoid(np.dot(X, W) + bias).reshape((X.shape[0],))

    def SGD_optimizer(self,X,Y,W,hypothesis,bias,alpha,batch_size):
        #making...
        sampled_X,sampled_Y = shuffle(X,Y)
        sampled_X = sampled_X[0:batch_size]
        sampled_Y = sampled_Y[0:batch_size]
        W = W - alpha * self.Round_H_Round_W(sampled_X, hypothesis, sampled_Y)
        bias = bias - alpha * self.Round_H_Round_bias(sampled_X, hypothesis, sampled_Y)
        acc = self.accuracy(hypothesis, sampled_Y)
        return W, bias, acc

    def GD_optimizer(self,X,Y,W,hypothesis,bias, alpha):
        W = W - alpha * self.Round_H_Round_W(X, hypothesis, Y)
        bias = bias - alpha * self.Round_H_Round_bias(X, hypothesis, Y)
        acc = self.accuracy(hypothesis, Y)
        return W, bias, acc

    def accuracy(self,hypothesis,Y):
        predict = np.round(hypothesis)
        correct = [1 if predict[i] == Y[i] else 0 for i in range(len(Y))]
        acc = np.sum(correct) / len(correct)
        return acc

def divide_data_into_0_1(combined_data):
    temp = []
    for i in combined_data:
        if int(i[-1]) != 2:
            temp += [i]
    a =  np.array(temp)
    b = np.split(a, [-1], axis=1)
    X = b[0]
    Y = b[1].astype(np.int32)
    Y = np.squeeze(Y)  # array shape (number,1) >> (number,)

    return X,Y

def normalize(X):
    mean = np.mean(X, axis=0, dtype=np.float32)
    std = np.std(X, axis=0, dtype=np.float32)
    X = (X - mean) / std
    return X

def sigmoid(X):
    return 1/(1 + np.exp(-X))

if __name__ == '__main__':

    # data preparing
    a = load_iris()
    target = a['target']
    target_names = a['target_names']  ## setosa == 0 , versicolor == 1 , virginica == 2
    features = a['data']  ## 4 feature
    #np.random.seed(0)
    features, target = shuffle(features, target)  # fixed by np.random.seed

    feature_for_train = features[:130]
    target_for_train = target[:130].reshape(130,1)
    feature_for_test = features[130:]
    target_for_test = target[130:].reshape(150-130,1)

    #######################################
    #decision tree for specific case {0,1,2}
    #######################################

    #train for first decision {0,1} || {2}
    X = feature_for_train
    Y = np.array([0 if i == 0 or i == 1 else 1 for i in target_for_train])
    W1,bias1 = Binary_inquiry_train(X,Y).output
    print(X.shape,Y.shape)

    #train for second decision {0} || {1}
    combined_data = np.concatenate((feature_for_train, target_for_train), axis=1)
    X, Y = divide_data_into_0_1(combined_data)
    print(X.shape, Y.shape)
    W2,bias2 = Binary_inquiry_train(X,Y).output

    #test
    test_X = normalize(feature_for_test)
    test_Y = target_for_test

    acc = 0
    for i,X in enumerate(test_X):

        hypothesis = sigmoid(np.dot(X,W1) + bias1)
        predict = np.squeeze(np.round(hypothesis).astype(np.int32))

        if predict == 1:
            print("ans = {} , prediction = {}".format(test_Y[i][0], 2))
            acc += test_Y[i] == 2
        else:
            hypothesis = sigmoid(np.dot(X, W2) + bias2)
            predict = np.squeeze(np.round(hypothesis).astype(np.int32))

            if predict ==1:
                print("ans = {} , prediction = {}".format(test_Y[i][0], 1))
                acc += test_Y[i] == 1

            else:
                print("ans = {} , prediction = {}".format(test_Y[i][0], 0))
                acc += test_Y[i] == 0

    acc = acc / len(test_Y)
    print("acc = {} %".format(acc[0]*100))


from file_class import File
import datetime

class TextFile(File):

    def __init__(self, name: str, file_type: str, size: str, description: str, content: str):
        super().__init__(name, file_type, size, description, content)
        self.num_of_characters = len(self.content)

    def write(self, new_content):
        self.content = new_content
        self.num_of_characters = len(new_content)
        self.date = datetime.datetime.now().date()
        self.time = datetime.datetime.now().time()


import csv

from function_store.path_settings import FILES_PATH_csv

def average():
    with open(FILES_PATH_csv, newline="") as file:
        reader = csv.DictReader(file)
        average_height = 0
        average_weight = 0
        row_counter = 0
        for row in reader:
            average_height += float(row[' "Height(Inches)"'])
            average_weight += float(row[' "Weight(Pounds)"'])
            row_counter += 1

        return (
            f"
Average height is: {round(average_height / row_counter, 2)} kg
"
            f"
Average weight is: {round(average_weight / row_counter, 2)} cm
"
        )

if __name__ == "__main__":
    average()


import numpy as np

class CBVModel():
    """Count-based value model.
    """
    def __init__(self, game_version):
        """Initializes a little alchemy count-based value model.

        Args:
            game_version (str): 'joined', 'alchemy1', 'alchemy2', 'tinyalchemy' or 'tinypixels.
                        States what element and combination set is going to be used.
        """
        # get number of elements
        if game_version == 'joined':
            self.n_elements = 738
        elif game_version == 'alchemy1' or game_version == 'tinyalchemy' or game_version == 'tinypixels':
            self.n_elements = 540
        elif game_version == 'alchemy2':
            self.n_elements = 720

    def get_value(self, combination):
        """Returns value for given combination.

        Args:
            combination (list): List of element indices that are involved in combination.

        Returns:
            float: Value.
        """
        return self.values_elements[combination[0]] + self.values_elements[combination[1]]

    def reset(self):
        """Resets model dependent information.
        """
        self.values_elements = np.zeros(self.n_elements)

    def update_model_specifics(self, chosen_combination, results):
        """Updates model specifics.

        Args:
            chosen_combination (list): List of element indices that are involved in combination.
            result (tuple): Consists of
                        (1) new_results_non_final (list): List of non final element indices that resulted from combination.
                        (2) new_results_total (list): List of all new element indices that resulted from combination.
        """
        if len(results[1]) == 0:
            self.values_elements[chosen_combination[0]] -= 1
            self.values_elements[chosen_combination[1]] -= 1
        else:
            self.values_elements[chosen_combination[0]] += len(results[1])
            self.values_elements[chosen_combination[1]] += len(results[1])


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import re
import os

#run web driver headlessly
os.environ['MOZ_HEADLESS'] = '1'
binary = FirefoxBinary('/usr/bin/firefox')
driver = webdriver.Firefox(firefox_binary=binary)
advanced_csv = 'Rk,Player,Pos,Age,Tm,G,MP,PER,TS%,3PAr,FTr,ORB%,DRB%,TRB%,AST%,STL%,BLK%,TOV%,USG%,,OWS,DWS,WS,WS/48,,OBPM,DBPM,BPM,VORP,Year\n'
per_game_csv = 'Rk,Player,Pos,Age▼,Tm,G,GS,MP,FG,FGA,FG%,3P,3PA,3P%,2P,2PA,2P%,eFG%,FT,FTA,FT%,ORB,DRB,TRB,AST,STL,BLK,TOV,PF,PS/G,Year\n'

#iterate all advanced stats
for year in range(1950,2018):
    driver.get('https://www.basketball-reference.com/leagues/NBA_'+str(year)+'_advanced.html')
    driver.execute_script('document.querySelector("#advanced_stats_toggle_partial_table").click()')
    driver.execute_script('document.querySelector("#all_advanced_stats > div:nth-child(1) > div:nth-child(3) > ul:nth-child(1) > li:nth-child(1) > div:nth-child(2) > ul:nth-child(1) > li:nth-child(4) > button:nth-child(1)").click()')
    csv = driver.find_element_by_css_selector("#csv_advanced_stats").text.split('\n',1)[-1]

    #add year to each row, clean up formatting
    csv = csv.replace('\n', ',' + str(year) + '\n')
    csv +=  ',' + str(year) + '\n'
    advanced_csv += csv

#iterate all per_game stats
for year in range(1950,2018):
    driver.get('https://www.basketball-reference.com/leagues/NBA_'+str(year)+'_per_game.html')
    driver.execute_script('document.querySelector("#per_game_stats_toggle_partial_table").click()')
    driver.execute_script('document.querySelector("#all_per_game_stats > div:nth-child(1) > div:nth-child(3) > ul:nth-child(1) > li:nth-child(1) > div:nth-child(2) > ul:nth-child(1) > li:nth-child(4) > button:nth-child(1)").click()')
    csv = driver.find_element_by_css_selector("#csv_per_game_stats").text.split('\n',1)[-1]

    #add year to each row, clean up formatting
    csv = csv.replace('\n', ',' + str(year) + '\n')
    csv +=  ',' + str(year) + '\n'
    per_game_csv += csv

#write to csv
stats = open("advanced_stats.csv", "w")
stats.write(advanced_csv)
stats.close()
stats = open("per_game_stats.csv", "w")
stats.write(per_game_csv)
stats.close()


import openpyxl
import sys

def create_excel_file(output_file, data_list):
    # Create a new workbook and select the active sheet
    workbook = openpyxl.Workbook()
    sheet = workbook.active

    # Write each item in the list to a new cell in the first column
    for index, item in enumerate(data_list, start=1):
        sheet.cell(row=index, column=1, value=item)

    # Save the workbook to the specified output file
    workbook.save(output_file)

def read_list_from_file(file_path):
    with open(file_path, 'r') as file:
        lines = file.readlines()
        # Remove bullet points or hyphens and leading/trailing whitespaces
        cleaned_lines = [line.strip('- ').strip() for line in lines]
        return cleaned_lines

if __name__ == "__main__":
    # Check if the correct number of command-line arguments is provided
    if len(sys.argv) != 2:
        print("Usage: python script_name.py input_file.txt")
        sys.exit(1)

    # Get the file paths from command-line arguments
    input_file = sys.argv[1]
    output_file = 'output.xlsx'

    # Read the list from the file and create the Excel file
    data_list = read_list_from_file(input_file)
    create_excel_file(output_file, data_list)


# scale: Hagen's Matrices Test - Short form version
# scale source: Heydasch, T., Haubrich, J. & Renner, K.-H. (2020). The Short Version of the Hagen Matrices Test (HMT-S). A 6-Item Induction Intelligence Test (T. Heydasch, Trans.). Methods, Data, Analyses, MDA, 7, 183e-205e. https://doi.org/10.12758/mda.2013.021

import registerscales
import registerparticipants

class HMTS:
    def __init__(self):
        self.name = 'hmts'
        self.score = 0
        self.min = 0
        self.max = 6
        self.codes = {
            "hmts_q1_m1": 6,
            "hmts_q2_m2": 4,
            "hmts_q3_m3": 1,
            "hmts_q4_m5": 3,
            "hmts_q5_m7": 5,
            "hmts_q6_m9": 6,
        }

    # def addcolumn(self,colname):
    #     if colname not in registerscales.DPmScales().getscalecolumns(self.name):
    #         registerscales.DPmScales().registerscalecolumn(self.name,colname)

    def calculate(self,responsedict):
        templatecolumns = registerscales.DPmScales().getscalecolumns(self.name)
        columns = responsedict.items()
        count = len(templatecolumns)
        check = 0
        for colname in templatecolumns:
            for colname2,value in columns:
                if colname == colname2:
                    if value == '' or value.isspace(): continue

                    check += 1

        if check != count: self.score = check == 0 and registerscales.DPmNullFlags['empty'] or 0
        if self.score is registerscales.DPmNullFlags['empty']: return self

        # Calculation:
        for colname in templatecolumns:
            for colname2,value in columns:
                if colname != colname2: continue

                if value == '' or value.isspace(): continue

                if self.codes[colname] == int(value):
                    self.score += 1

        return self

    def report(self):
        print('HMT-S:',self.score,'/',self.max)

    def exportdata(self):
        return self.score

    def setprofile(self,pid):
        self.score = registerparticipants.DPmProfiles().getprofile(pid).getscaledata(self.name)

    def createtables(self,conn):
        conn.cursor().executescript('''
            DROP TABLE IF EXISTS HMTS;

            CREATE TABLE HMTS (
                pid  TEXT NOT NULL PRIMARY KEY UNIQUE,
                score INTEGER
             );''')

    def exporttodb(self,conn,pid):
        self.setprofile(pid)

        conn.cursor().execute('''INSERT OR REPLACE INTO HMTS (pid, score) VALUES (?, ?)''', (pid, self.score))
        conn.commit()

    def preparecsv(self,pid,IncludeHeader=True):
        self.setprofile(pid)

        headers = ['pid','hmts']
        rows = [
            [pid,self.score]
        ]

        return headers,rows

registerscales.DPmScales().registerscale(HMTS())


import math
from heapq import heappush, heappop

# -------------------insertion sort-------------------
# ""when we reach to an element we compare that element with others and find the correct place for that""
A = [3, 7, 4, 5, 0, -1, 9, 12, 10, 6, 3, 5, 6, -100]
for i in range(0, len(A)):
    item = A[i]
    k = i
    while k > 0 and A[k - 1] > item:
        A[k] = A[k - 1]
        k -= 1
    A[k] = item
print(A)
# -----------------------------------------------------

# ---------------------bubble sort---------------------
# ""from end compare each element with the last element if the last one was smaller we change them""
A = [3, 7, 4, 5, 0, -1, 9, 12, 10, 6, 3, 5, 6, -100]
n = len(A)
for j in range(n):
    bubble_found = False
    for i in range(n - 1, j, -1):
        if A[i] < A[i - 1]:
            A[i], A[i - 1] = A[i - 1], A[i]
            bubble_found = True
    if not bubble_found:
        break
print(A)
# ------------------------------------------------------

# -------------------counting sort----------------------
# ""count the number of every number has repeated""
A = [3, 7, 4, 5, 0, 9, 12, 10, 6, 3, 5, 6]
m = max(A)
Count = [0] * (m + 1)
for x in A:
    Count[x] += 1
Temp = []
for x in range(m + 1):
    Temp += [x] * Count[x]
print(Temp)
# -------------------------------------------------------

# -------------------selection sort----------------------
# ""chose the element and compare with next element and change that with the smallest one""
A = [3, 7, 4, 5, 0, 9, 12, 10, 6, 3, 5, 6]
num = len(A)
for i in range(num - 1):
    selection = i
    for j in range(i + 1, num):
        if A[j] < A[selection]:
            selection = j
    A[selection], A[i] = A[i], A[selection]
print(A)
# ---------------------------------------------------------

# --------------------merge sort---------------------------
# ""divide array to two array and continue this""
def mergeSort(array):
    if len(array) > 1:
        length = len(array)//2
        left = array[:length]
        right = array[length:]
        mergeSort(left)
        mergeSort(right)
        i = j = k = 0
        while i < len(left) and j < len(right):
            if left[i] < right[j]:
                array[k] = left[i]
                i += 1
            else:
                array[k] = right[j]
                j += 1
            k += 1
        while i < len(left):
            array[k] = left[i]
            i += 1
            k += 1
        while j < len(right):
            array[k] = right[j]
            j += 1
            k += 1
array = [3, 7, 4, 5, 0, 9, 12, 10, 6, 3, 5, 6]
mergeSort(array)
print(array)
# -----------------------------------------------------------

# ----------------------bucket sort--------------------------
# ""change the array to ten array and use bubble sort for that""
def bubble_sort(A):
    n = len(A)
    for j in range(n):
        bubble_found = False
        for i in range(n - 1, j, -1):
            if A[i] < A[i - 1]:
                A[i], A[i - 1] = A[i - 1], A[i]
                bubble_found = True
        if not bubble_found:
            break
    return (A)

def bucket_sort(A):
    B = [[] for i in range(10)]
    for i in A:
        B[i // 10] += [i]
    print(B)
    A = []
    for i in range(10):
        A += bubble_sort(B[i])
    return A
array = [3, 7, 4, 5, 0, 9, 12, 10, 6, 3, 5, 6, 52, 48, 42, 51, 96, 95, 95, 91, 36, 37]
print(bucket_sort(array))
# ---------------------------------------------------------------

# ---------------------binary search-----------------------------
# ""found middle of an array and check that is this that we want or not""
def binary_search(array ,x):
    up = len(array) - 1
    down = 0
    while up >= down:
        mid = (up + down)//2
        if array[mid] == x:
            return "found at", mid + 1
        elif array[mid] < x:
            down = mid + 1
        else:
            up = mid - 1
    return "not found"
myArray = [1, 5, 9, 15, 21, 59, 100, 652, 666, 897]
print(binary_search(myArray, 666))
# ------------------------------------------------------------------

# ------------------------lower bound-------------------------------
# ""found the last smaller of one element""
def lower_bound(a, x):
    if a[0] >= x:
        return -1
    start = 0
    end = len(a)
    while end > start + 1:
        mid = start + end >> 1
        if a[mid] >= x:
            end = mid
        else:
            start = mid
    return start
# ------------------------------------------------------------------

# ------------------------upper bound-------------------------------
# ""found the next bigger of one element""
def upper_bound(a, x):
    if a[len(a)-1] <= x:
        return -1
    start = 0
    end = len(a)
    while end > start + 1:
        mid = start + end >> 1
        if a[mid] > x:
            end = mid
        else:
            start = mid
    return end
# ------------------------------------------------------------------

# ------------------------intro sort--------------------------------
arr = []

def heapsort():
    global arr
    h = []
    for value in arr:
        heappush(h, value)
    arr = []
    arr = arr + [heappop(h) for i in range(len(h))]

def InsertionSort(begin, end):
    left = begin
    right = end
    for i in range(left + 1, right + 1):
        key = arr[i]
        j = i - 1
        while j >= left and arr[j] > key:
            arr[j + 1] = arr[j]
            j = j - 1
        arr[j + 1] = key

def Partition(low, high):
    global arr
    pivot = arr[high]
    i = low - 1
    for j in range(low, high):
        if arr[j] <= pivot:
            i = i + 1
            (arr[i], arr[j]) = (arr[j], arr[i])
    (arr[i + 1], arr[high]) = (arr[high], arr[i + 1])
    return i + 1

def MedianOfThree(a, b, d):
    global arr
    A = arr[a]
    B = arr[b]
    C = arr[d]
    if A <= B and B <= C:
        return b
    if C <= B and B <= A:
        return b
    if B <= A and A <= C:
        return a
    if C <= A and A <= B:
        return a
    if B <= C and C <= A:
        return d
    if A <= C and C <= B:
        return d

def IntrosortUtil(begin, end, depthLimit):
    global arr
    size = end - begin
    if size < 16:
        InsertionSort(begin, end)
        return
    if depthLimit == 0:
        heapsort()
        return
    pivot = MedianOfThree(begin, begin + size // 2, end)
    (arr[pivot], arr[end]) = (arr[end], arr[pivot])
    partitionPoint = Partition(begin, end)
    IntrosortUtil(begin, partitionPoint - 1, depthLimit - 1)
    IntrosortUtil(partitionPoint + 1, end, depthLimit - 1)

def Introsort(begin, end):
    depthLimit = 2 * math.floor(math.log2(end - begin))
    IntrosortUtil(begin, end, depthLimit)
# ------------------------------------------------------------------