# -*- coding: utf-8 -*-
"""
Created on Tue Oct 27 08:37:30 2020

@author: pedro-salj
"""

#############################
##### CONFIGURAÇÃO GERAL ####
#############################
globals().clear()
""" Mudar diretório """
import os
from pathlib import Path
import getpass
if getpass.getuser() == "pedro":
    print('\nLogado de casa')
    caminho_base = Path(r'D:\Códigos, Dados, Documentação e Cheat Sheets')
elif getpass.getuser() == "pedro-salj":
    print('\nLogado da salj-alems')
    caminho_base = Path(r'C:\Users\pedro-salj\Desktop\Pedro Nakashima\Códigos, Dados, Documentação e Cheat Sheets')

""" Mudar diretório para dados Siconfi"""
caminho_wd = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python Time Series (Udemy)' / 'Data'
print('\nDiretório anterior:\n', os.getcwd())
os.chdir(caminho_wd)
print('\nDiretório atual:\n', os.getcwd())

#############################
#############################
#############################
globals().clear()

from datetime import datetime

my_year = 2020
my_month = 1
my_day = 2
my_hour = 13
my_min = 30
my_sec = 15

my_date = datetime(my_year, my_month, my_day)
print(my_date)
print(type(my_date))

my_date_time = datetime(my_year, my_month, my_day, my_hour, my_min, my_sec)
print(my_date_time)

print(my_date_time.day)

#############################
##### datetime do numpy #####
#############################

import numpy as np

date = np.array(['2020-03-15', '2020-03-16', '2020-03-17'])
print(type(date))

date = np.array(['2020-03-15', '2020-03-16', '2020-03-17'], dtype='datetime64')
print(date.dtype)

date = np.array(['2020-03-15', '2020-03-16', '2020-03-17'], dtype='datetime64[Y]')
print(date.dtype)

date = np.array(['2020-03-15', '2020-03-16', '2020-03-17'], dtype='datetime64[h]')
print(date.dtype)
print(date)

#############################
##### np.arange #############
#############################
print(np.arange(0,10,2))

print(np.arange('2018-06-01','2018-06-23', 7, dtype='datetime64[D]'))

print(np.arange('1968','1976', dtype='datetime64[Y]'))

#############################
##### pandas ################
#############################

import pandas as pd

print(pd.date_range('2020-01-01', periods=7, freq='D'))

print(pd.date_range('Jan 01, 2018', periods=7, freq='D'))

print(pd.to_datetime(['1/2/2018', 'Jan 03, 2018']))

print(pd.to_datetime(['1/2/2018', '1/3/2018']))

print(pd.to_datetime(['2/1/2018', '3/1/2018']))

print(pd.to_datetime(['2/1/2018', '3/1/2018'], format='%d/%m/%Y'))

print(pd.to_datetime(['2--1--2018', '3--1--2018'], format='%d--%m--%Y'))

data = np.random.randn(3,2)
cols = ['A', 'B']
print(data)

idx = pd.date_range('2020-01-01', periods=3, freq='D')

df = pd.DataFrame(data, index=idx, columns=cols)

print(df)
print(df.index)
print(df.index.max())
print(df.index.argmax())
print(df.index.argmin())

#############################
##### Time Resampling #######
#############################

import pandas as pd

pasta = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python for time series (Udemy)' / 'Data'
df = pd.read_csv(pasta / 'starbucks.csv', index_col='Date', parse_dates=True)

print(df.head())

print(df.info())

print(df.index)

# daily --> yearly
print(df.resample(rule='A').mean())
print(df.resample(rule='A').std())

def first_day(entry):

    # IS THERE AN ENTRY?
    if len(entry) != 0:
        # IF SO, RETURN FIRST ENTRY
        return entry[0]

print(df.resample(rule='A').apply(first_day))

df['Close'].resample('A').mean().plot.bar()

title = 'Monthly Max Closing Price for Starbucks'
df['Close'].resample('M').max().plot.bar(figsize=(16,6), title=title, color='#1f77b4')

#############################
##### Time Shifting #########
#############################

import pandas as pd

pasta = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python for time series (Udemy)' / 'Data'
df = pd.read_csv(pasta / 'starbucks.csv', index_col='Date', parse_dates=True)

print(df.head())
print(df.tail())

print(df.shift(1))

print(df.shift(1, fill_value=0))

print(df.shift(1).tail())

print(df.shift(5))

# df = df.shift(5)

print(df.shift(-1))
print(df.shift(-1).tail())

# -----------------------------------

print(df.shift(periods=1, freq='M'))

#################################
##### Rolling and Expanding #####
#################################

# Rolling ---------------------------------------

import pandas as pd

pasta = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python for time series (Udemy)' / 'Data'
df = pd.read_csv(pasta / 'starbucks.csv', index_col='Date', parse_dates=True)

df['Close'].plot()

df['Close'].plot(figsize=(12,5))

print(df.rolling(window=7).mean().head(10))

df['Close'].plot(figsize=(12,5))
df.rolling(window=7).mean()['Close'].plot()

df['Close'].plot(figsize=(12,5))
df.rolling(window=30).mean()['Close'].plot()

df['Close'].plot(figsize=(12,5))
df.rolling(window=60).mean()['Close'].plot()

df['Close'].plot(figsize=(12,5))
df.rolling(window=15).mean()['Close'].plot()

df['Close: 30 Day Mean'] = df['Close'].rolling(window=30).mean()

print(df.head())

df[['Close', 'Close: 30 Day Mean']].plot(figsize=(12,5))

# Expanding ---------------------------------------

df['Close'].expanding().mean().plot(figsize=(12,5))

########################################
##### Visualizing Time Series Data #####
########################################

import pandas as pd

pasta = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python for time series (Udemy)' / 'Data'
df = pd.read_csv(pasta / 'starbucks.csv', index_col='Date', parse_dates=True)

print(df.index)

#df.index = pd.to_datetime(df.index)

df.plot()

df['Close'].plot()

df['Close'].plot();

df['Volume'].plot()

title = 'TITLE'
ylabel = 'Y LABEL'
xlabel = 'X LABEL'

df['Close'].plot(figsize=(12,6), title=title)

""" Para adicionar labels x e y, precisa atribuir o
gráfico a uma variável, por convenção ax """

ax = df['Close'].plot(figsize=(12,6), title=title)

ax.autoscale(axis='both', tight=True) # sem espaço em branco na vertical e horizontal

ax.set(xlabel=xlabel, ylabel=ylabel)

# ---------------------

ax = df['Close'].plot(figsize=(12,6), title=title)

ax.autoscale(axis='y', tight=True) # sem espaço em branco na vertical

ax.set(xlabel=xlabel, ylabel=ylabel)

# ---------------------

""" Plotando apenas um ano. Forma 1 """

print(df['Close'])
print(df['Close']['2017-01-01':'2017-12-31'])

df['Close'].plot(figsize=(12,4))

df['Close']['2017-01-01':'2017-12-31'].plot(figsize=(12,4))

""" Plotando apenas um ano. Forma 2 """

df['Close'].plot(figsize=(12,4), xlim = ['2017-01-01','2017-12-31'])

""" Usando ylim """

df['Close']['2017-01-01':'2017-12-31'].plot(figsize=(12,4), ylim=[0,70])

df['Close']['2017-01-01':'2017-12-31'].plot(figsize=(12,4), ylim=[40,70])

""" Estilo da linha: ls """

df['Close']['2017-01-01':'2017-12-31'].plot(figsize=(12,4), ylim=[40,70], ls='--')

""" Cor da linha: c """

df['Close']['2017-01-01':'2017-12-31'].plot(figsize=(12,4), ylim=[40,70], ls='--', c='red')

""" Outras configurações """

import pandas as pd

pasta = caminho_base / 'Dados' / 'Cursos e Livros' / 'Python for time series (Udemy)' / 'Data'
df = pd.read_csv(pasta / 'starbucks.csv', index_col='Date', parse_dates=True)

df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60])

df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))

""" tirando o label do eixo x """

ax = df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))
ax.set(xlabel='')

""" xticks
Agora, as datas estão igualmente espaçadas
Agora, não há mais marcação de cada novo mês
Agora, o espaçamento é por 1 semana"""

from matplotlib import dates

ax = df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))
ax.set(xlabel='')

ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=0))

""" formatando as datas"""

from matplotlib import dates

ax = df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))
ax.set(xlabel='')

ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=0))
ax.xaxis.set_major_formatter(dates.DateFormatter('%a-%B-%d'))

""" minor axis """

from matplotlib import dates

ax = df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))
ax.set(xlabel='')

ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=0))
ax.xaxis.set_major_formatter(dates.DateFormatter('%d'))

ax.xaxis.set_minor_locator(dates.MonthLocator())
ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%b')) #\n representa uma nova linha

""" grid """

from matplotlib import dates

ax = df['Close'].plot(xlim=['2017-01-01','2017-03-01'], ylim=[50,60], figsize=(12,5))
ax.set(xlabel='')

ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=0))
ax.xaxis.set_major_formatter(dates.DateFormatter('%d'))

ax.xaxis.set_minor_locator(dates.MonthLocator())
ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%b')) #\n representa uma nova linha

ax.yaxis.grid(True)
ax.xaxis.grid(True)

dt_str = '202001'
print(pd.to_datetime([dt_str], format='%Y%m'))

Cursos e Vídeos/Python/Python for Time Series (Udemy)/6 Time series.py


"""
Provide registration functions for numerical grid parameters.

This module handles grid parameters and grid functions across various NRPy-supported infrastructures.

Author: Zachariah B. Etienne
        zachetie **at** gmail **dot* com
"""
from typing import List, Any, Union, Dict, Tuple, Optional, Sequence
import sympy as sp
import nrpy.indexedexp as ixp
import nrpy.params as par

Cart_origin = par.register_CodeParameters(
    "REAL", __name__, ["Cart_originx", "Cart_originy", "Cart_originz"], 0.0
)
_ = par.register_CodeParameter("int", __name__, "NUMGRIDS", 1, commondata=True)

# Core GridFunction class.
class GridFunction:
    """The core class for grid functions."""

    def __init__(self, name: str, group: str = "EVOL") -> None:
        self.c_type_alias: str = "double"
        self.name: str = name
        self.group: str = group

    def read_gf_from_memory_Ccode_onept(
        self, i0_offset: int = 0, i1_offset: int = 0, i2_offset: int = 0, **kwargs: Any
    ) -> str:
        """Catch-all function for gridfunctions that haven't been set up correctly."""
        return f"Please define read_gf_from_memory_Ccode_onept() inside grid.py. Inputs: [class instance] {i0_offset} {i1_offset} {i2_offset} {kwargs}"

class BHaHGridFunction(GridFunction):
    """The subclass for BlackHoles@Home grid functions."""

    def __init__(
        self,
        name: str,
        group: str = "EVOL",
        rank: int = 0,
        f_infinity: float = 0.0,
        wavespeed: float = 1.0,
        is_basename: bool = True,
        gf_array_name: str = "use_in_gfs_for_EVOL_auxevol_gfs_for_AUXEVOL_etc",
    ) -> None:
        super().__init__(name, group)
        self.c_type_alias = "REAL"  # always use REAL
        self.rank = rank
        self.f_infinity = f_infinity
        self.wavespeed = wavespeed
        self.is_basename = is_basename
        self.verify_gridfunction_group_is_valid()
        if gf_array_name == "use_in_gfs_for_EVOL_auxevol_gfs_for_AUXEVOL_etc":
            if group == "EVOL":
                self.gf_array_name = "in_gfs"
            elif group == "AUXEVOL":
                self.gf_array_name = "auxevol_gfs"
            elif group == "AUX":
                self.gf_array_name = "aux_gfs"
        else:
            self.gf_array_name = gf_array_name

        if is_basename:
            self.verify_gridfunction_basename_is_valid(self.name)

    def verify_gridfunction_basename_is_valid(self, name: str) -> None:
        """
        Validate the gridfunction's base name.

        Raises a ValueError if base name is zero-length or ends with an integer.
        Raises TypeError if the base name is not a string.

        Gridfunction base names must not end with integers. For instance, a rank-1
        gridfunction 'vecU1' has a valid base name 'vecU'. A scalar gridfunction
        named 'u2' would have an invalid base name 'u2'. This strict requirement
        facilitates quick identification of a gridfunction by its name.

        :param name: The name of the gridfunction.

        Raises
        ------
        ValueError
            If the name is of zero length or ends with an integer.
        TypeError
            If the name is not of type str.

        Example:
            >>> try: GF = BHaHGridFunction(name="h1")
            ... except: "Attempt to create a gridfunction ending in an integer failed. Good."
            'Attempt to create a gridfunction ending in an integer failed. Good.'
        """
        # First check for zero-length basenames:
        if len(name) == 0:
            raise ValueError("Tried to register gridfunction without a name!")

        if not isinstance(name, str):
            raise TypeError("Gridfunction names must be strings")

        if len(name) > 0 and name[-1].isdigit():
            raise ValueError(
                f"Tried to register gridfunction with base name: {name}\n"
                f"To ensure that finite difference code generations do not get confused,\n"
                f"gridfunctions with base names ending in an integer are forbidden; pick a new name."
            )

    def verify_gridfunction_group_is_valid(self) -> None:
        """
        Validate the gridfunction group.

        The valid groups are 'EVOL', 'AUX' and 'AUXEVOL'.

        'EVOL': for evolved quantities (i.e., quantities stepped forward in time),
        'AUXEVOL': for auxiliary quantities needed at all points by evolved quantities,
        'AUX': for all other quantities needed at all gridpoints.

        :raises ValueError: If the group is not valid.

        Doctests:
        >>> try: GF = BHaHGridFunction(name="blah", group="INVALID")
        ... except: print("Errored out. This is good.")
        Errored out. This is good.
        """
        valid_groups = (
            "EVOL",
            "AUX",
            "AUXEVOL",
        )
        if self.group not in valid_groups:
            msg = (
                f"Unsupported gridfunction group {self.group}. Supported groups include:\n"
                '    "EVOL": for evolved quantities (i.e., quantities stepped forward in time),\n'
                '    "AUXEVOL": for auxiliary quantities needed at all points by evolved quantities,\n'
                '    "AUX": for all other quantities needed at all gridpoints.\n'
            )
            raise ValueError(msg)

    def read_gf_from_memory_Ccode_onept(
        self, i0_offset: int = 0, i1_offset: int = 0, i2_offset: int = 0, **kwargs: Any
    ) -> str:
        """
        Retrieve a grid function value from memory for a given offset.

        :param i0_offset: Offset for the first index.
        :param i1_offset: Offset for the second index.
        :param i2_offset: Offset for the third index.
        :param kwargs: Optional keyword arguments, expected to have 'gf_array_name'.

        :return: Formatted string.

        :raises ValueError: If 'gf_array_name' is not provided in kwargs.

        Doctests:
        >>> glb_gridfcs_dict.clear()
        >>> par.set_parval_from_str("Infrastructure", "BHaH")
        >>> abc = register_gridfunctions("abc")
        >>> glb_gridfcs_dict["abc"].read_gf_from_memory_Ccode_onept(1, 2, 3)
        'in_gfs[IDX4(ABCGF, i0+1, i1+2, i2+3)]'
        >>> defg = register_gridfunctions("defg")
        >>> glb_gridfcs_dict["defg"].read_gf_from_memory_Ccode_onept(0, -1, 0, enable_simd=True, gf_array_name="My_Array")
        'ReadSIMD(&My_Array[IDX4(DEFGGF, i0, i1-1, i2)])'
        """
        i0 = f"i0+{i0_offset}".replace("+-", "-") if i0_offset != 0 else "i0"
        i1 = f"i1+{i1_offset}".replace("+-", "-") if i1_offset != 0 else "i1"
        i2 = f"i2+{i2_offset}".replace("+-", "-") if i2_offset != 0 else "i2"

        gf_array_name = kwargs.get("gf_array_name", self.gf_array_name)

        ret_string = f"{gf_array_name}[IDX4({self.name.upper()}GF, {i0}, {i1}, {i2})]"
        if kwargs.get("enable_simd"):
            return f"ReadSIMD(&{ret_string})"
        return ret_string

    @staticmethod
    def access_gf(
        gf_name: str,
        i0_offset: int = 0,
        i1_offset: int = 0,
        i2_offset: int = 0,
        gf_array_name: str = "in_gfs",
    ) -> str:
        """
        Retrieve a grid function value from memory for a given offset.

        :param gf_name: The grid function name.
        :param i0_offset: Offset for the first index.
        :param i1_offset: Offset for the second index.
        :param i2_offset: Offset for the third index.
        :param gf_array_name: Optional grid function array name.

        :return: Formatted string.

        :raises ValueError: If 'gf_array_name' is not provided.

        Doctests:
        >>> BHaHGridFunction.access_gf("aa", 1,2,3)
        'in_gfs[IDX4(AAGF, i0+1, i1+2, i2+3)]'
        >>> BHaHGridFunction.access_gf("defg", 0, -1, 0, "My_Array")
        'My_Array[IDX4(DEFGGF, i0, i1-1, i2)]'
        """
        i0 = f"i0+{i0_offset}".replace("+-", "-") if i0_offset != 0 else "i0"
        i1 = f"i1+{i1_offset}".replace("+-", "-") if i1_offset != 0 else "i1"
        i2 = f"i2+{i2_offset}".replace("+-", "-") if i2_offset != 0 else "i2"

        return f"{gf_array_name}[IDX4({gf_name.upper()}GF, {i0}, {i1}, {i2})]"

    @staticmethod
    def gridfunction_lists() -> Tuple[List[str], List[str], List[str]]:
        """
        Generate sorted lists of gridfunction names for each group type: 'EVOL', 'AUX', and 'AUXEVOL'.

        This function creates a dictionary with group types as keys and corresponding gridfunction names
        as values. It iterates through a global dictionary of gridfunctions, 'glb_gridfcs_dict', and
        appends the gridfunction names to the corresponding group lists in 'groups'. The function then
        sorts each list and returns the sorted lists as a tuple.

        Note: This function assumes the existence of a global gridfunction dictionary named
        'glb_gridfcs_dict' whose keys are gridfunction names and values are gridfunction objects. Each
        gridfunction object should have 'group' and 'name' attributes.

        :returns: A tuple containing lists of gridfunction names
                  for each group: 'EVOL', 'AUX', and 'AUXEVOL', respectively.

        Example:
            gridfunction_lists()
            (['evol_gf1', 'evol_gf2'], ['aux_gf1', 'aux_gf2'], ['auxevol_gf1', 'auxevol_gf2'])
        """
        # Initialize dictionary for holding lists of gridfunction names for each group.
        groups: Dict[str, List[str]] = {
            "EVOL": [],
            "AUX": [],
            "AUXEVOL": [],
        }

        # Iterate through the global dictionary of gridfunctions.
        for _key, gf in glb_gridfcs_dict.items():
            if gf.group in groups:
                groups[gf.group].append(gf.name)

        # Sort the lists. Iterating through a copy of the keys to avoid modifying the dictionary while iterating.
        for group in list(groups.keys()):
            groups[group] = sorted(groups[group])

        # Pack the sorted lists into a tuple and return.
        return groups["EVOL"], groups["AUX"], groups["AUXEVOL"]

    @staticmethod
    def gridfunction_defines() -> str:
        """
        Generate a string representation of all the GridFunction definitions.

        This function also includes definitions for the values of each gridfunction
        at infinity and the characteristic wave speed. The output is formatted to be
        used in C-style languages.

        This function assumes the existence of a global gridfunction dictionary named
        'glb_gridfcs_dict'. The keys of this dictionary should be gridfunction names,
        and the values should be instances of classes derived from GridFunction.

        :return: The resulting string representation of the GridFunction definitions.
        """

        def define_gfs(name: str, gfs: List[str]) -> str:
            """
            Generate string representation for a list of GridFunctions.

            :param name: The name of the group.
            :param gfs: The list of GridFunctions.
            :return: The resulting string representation.
            """
            num_gfs = len(gfs)
            defines = "\n".join(
                f"#define {gf.upper()}GF\t{i}" for i, gf in enumerate(gfs)
            )
            return f"\n// {name.upper()} VARIABLES:\n#define NUM_{name.upper()}_GFS {num_gfs}\n{defines}\n"

        (
            evolved_variables_list,
            auxiliary_variables_list,
            auxevol_variables_list,
        ) = BHaHGridFunction.gridfunction_lists()

        outstr = f"{define_gfs('EVOL', evolved_variables_list)}"

        if evolved_variables_list:
            # mypy: doesn't consider that this func is only called for BHaHGridFunctions.
            f_infinity_list = [str(glb_gridfcs_dict[var].f_infinity) for var in evolved_variables_list]  # type: ignore
            f_wavespeed_list = [str(glb_gridfcs_dict[var].wavespeed) for var in evolved_variables_list]  # type: ignore

            f_infinity_str = ", ".join(f_infinity_list)
            f_wavespeed_str = ", ".join(f_wavespeed_list)

            outstr += "\n\n// SET gridfunctions_f_infinity[i] = evolved gridfunction i's value in the limit r->infinity:\n"
            outstr += f"static const REAL gridfunctions_f_infinity[NUM_EVOL_GFS] = {{ {f_infinity_str} }};\n"

            outstr += "\n\n// SET gridfunctions_wavespeed[i] = evolved gridfunction i's characteristic wave speed:\n"
            outstr += f"static const REAL gridfunctions_wavespeed[NUM_EVOL_GFS] = {{ {f_wavespeed_str} }};\n"

        outstr += f"{define_gfs('AUX', auxiliary_variables_list)}"
        outstr += f"{define_gfs('AUXEVOL', auxevol_variables_list)}"

        return outstr

class BaseETGridFunction(GridFunction):
    """Subclass for basic (non-CarpetX) Einstein Toolkit grid functions."""

    def __init__(
        self,
        name: str,
        group: str = "EVOL",
        rank: int = 0,
        f_infinity: float = 0.0,
        wavespeed: float = 1.0,
        is_basename: bool = True,
    ) -> None:
        super().__init__(name, group)
        self.rank = rank
        self.f_infinity = f_infinity
        self.wavespeed = wavespeed
        self.is_basename = is_basename
        self.c_type_alias = "CCTK_REAL"

    def verify_gridfunction_group_is_valid(self) -> None:
        """
        Verify that the 'group' attribute of the GridFunction instance is one of the valid groups.

        The valid groups are:
        'EVOL': for evolved quantities (i.e., quantities stepped forward in time),
        'AUXEVOL': for auxiliary quantities needed at all points by evolved quantities,
        'AUX': for all other quantities needed at all gridpoints.

        :raises ValueError: If the 'group' attribute is not one of the valid groups.
        """
        valid_groups = (
            "EVOL",
            "AUX",
            "AUXEVOL",
        )
        if self.group not in valid_groups:
            msg = (
                f"Unsupported gridfunction group {self.group}. Supported groups include:\n"
                '    "EVOL": for evolved quantities (i.e., quantities stepped forward in time),\n'
                '    "AUXEVOL": for auxiliary quantities needed at all points by evolved quantities,\n'
                '    "AUX": for all other quantities needed at all gridpoints.\n'
            )
            raise ValueError(msg)

    def read_gf_from_memory_Ccode_onept(
        self, i0_offset: int = 0, i1_offset: int = 0, i2_offset: int = 0, **kwargs: Any
    ) -> str:
        """
        Generate a formatted string using the grid function name and offsets.

        If 'gf_array_name' is not provided in kwargs, a ValueError is raised.

        :param i0_offset: Offset for the first index
        :param i1_offset: Offset for the second index
        :param i2_offset: Offset for the third index
        :param kwargs: Optional keyword arguments.
        :return: Formatted string
        :raises ValueError: If 'gf_array_name' is not provided in kwargs or if kwargs are passed when they shouldn't be.

        Doctests:
        >>> glb_gridfcs_dict.clear()
        >>> par.set_parval_from_str("Infrastructure", "BaseET")
        >>> abc = register_gridfunctions("abc", group="EVOL")
        >>> glb_gridfcs_dict["abc"].read_gf_from_memory_Ccode_onept(1, 2, 3)
        'abc[CCTK_GFINDEX3D(cctkGH, i0+1, i1+2, i2+3)]'
        >>> defg = register_gridfunctions("defg", group="EVOL")
        >>> glb_gridfcs_dict["defg"].read_gf_from_memory_Ccode_onept(0, -1, 0)
        'defg[CCTK_GFINDEX3D(cctkGH, i0, i1-1, i2)]'
        """
        if kwargs:
            raise ValueError(
                "BaseETGridFunction.read_gf_from_memory_Ccode_onept() does not accept kwargs!"
            )

        i0 = f"i0+{i0_offset}".replace("+-", "-") if i0_offset != 0 else "i0"
        i1 = f"i1+{i1_offset}".replace("+-", "-") if i1_offset != 0 else "i1"
        i2 = f"i2+{i2_offset}".replace("+-", "-") if i2_offset != 0 else "i2"

        return f"{self.name}[CCTK_GFINDEX3D(cctkGH, {i0}, {i1}, {i2})]"

class CarpetXGridFunction(GridFunction):
    """Class for CarpetX gridfunction operations."""

    def __init__(
        self,
        name: str,
        group: str = "",
        rank: int = 0,
        f_infinity: float = 0.0,
        wavespeed: float = 1.0,
        is_basename: bool = True,
        centering: str = "C",
    ) -> None:
        super().__init__(name, group)
        self.rank = rank
        self.f_infinity = f_infinity
        self.wavespeed = wavespeed
        self.is_basename = is_basename
        self.centering = centering

        group_suffixes = {
            "EXTERNAL": "_ext",
            "CORE": "_core",
            "TILE_TMP": "_tile_tmp",
            "SCALAR_TMP": "",
        }
        if self.group in group_suffixes:
            self.name += group_suffixes[self.group]

        # Dictionaries for storing grid function groups.
        # self.index_group = {}
        # self.rev_index_group = {}

        # Validation step.
        self.verify_gridfunction_centering_is_valid()
        self.verify_gridfunction_group_is_valid()

    def verify_gridfunction_group_is_valid(self) -> None:
        """
        Verify that the 'group' attribute of the GridFunction instance is one of the valid groups.

        The valid groups are:
        'EVOL': for evolved quantities (i.e., quantities stepped forward in time),
        'AUXEVOL': for auxiliary quantities needed at all points by evolved quantities,
        'AUX': for all other quantities needed at all gridpoints,
        'EXTERNAL': for all quantities defined in other modules,
        'CORE': for all quantities defined inside the CarpetX driver,
        'TILE_TMP': for all temporary quantities defined for CarpetX tiles,
        'SCALAR_TMP': for all temporary quantities defined for doubles.

        :raises ValueError: If the 'group' attribute is not one of the valid groups.
        """
        valid_groups = (
            "EVOL",
            "AUX",
            "AUXEVOL",
            "EXTERNAL",
            "CORE",
            "TILE_TMP",
            "SCALAR_TMP",
        )
        if self.group not in valid_groups:
            msg = (
                f"Unsupported gridfunction group {self.group}. Supported groups include:\n"
                '    "EVOL": for evolved quantities (i.e., quantities stepped forward in time),\n'
                '    "AUXEVOL": for auxiliary quantities needed at all points by evolved quantities,\n'
                '    "AUX": for all other quantities needed at all gridpoints.\n'
                '    "EXTERNAL": for all quantities defined in other modules.\n'
                '    "CORE": for all quantities defined inside the CarpetX driver.\n'
                '    "TILE_TMP": for all temporary quantities defined for CarpetX tiles.\n'
                '    "SCALAR_TMP": for all temporary quantities defined for doubles.'
            )
            raise ValueError(msg)

    def verify_gridfunction_centering_is_valid(self) -> None:
        """
        Verify that the 'centering' attribute of the GridFunction instance is valid.

        A valid 'centering' is a 3-character string where each character is either 'C' or 'V'.
        'C' stands for cell-centered quantities, 'V' stands for vertex-centered quantities.

        :raises ValueError: If the 'centering' attribute is not valid.
        """
        if len(self.centering) != 3 or any(c not in ("C", "V") for c in self.centering):
            msg = (
                f"Unsupported gridfunction centering {self.centering}. Supported centerings are 3-character strings, each character being either:\n"
                '    "C": for cell-centered quantities,\n'
                '    "V": for vertex-centered quantities.'
            )
            raise ValueError(msg)

# Contains a list of gridfunction objects.
glb_gridfcs_dict: Dict[
    str, Union[GridFunction, BHaHGridFunction, BaseETGridFunction, CarpetXGridFunction]
] = {}

#####################################
# Outside the classes above, only define functions in grid.py that are used by multiple infrastructures

def register_gridfunctions(
    names: Union[str, List[str]], **kwargs: Any
) -> List[sp.Symbol]:
    """
    Register grid functions with a specified name or list of names.

    :param names: A name or list of names of grid functions to be registered.
    :param kwargs: Additional keyword arguments.
    :return: A list of sympy symbols representing the registered grid functions.

    Doctests:
    >>> glb_gridfcs_dict.clear()
    >>> gridfunc = register_gridfunctions('gridfunc')[0]
    >>> print(gridfunc)
    gridfunc
    >>> # glb_gridfcs_dict["gridfunc"] is of type BHaHGridFunction. __dict__.items() access all variables in the class.
    >>> for key, value in glb_gridfcs_dict["gridfunc"].__dict__.items():
    ...     print(key, value)
    c_type_alias CCTK_REAL
    name gridfunc
    group EVOL
    rank 0
    f_infinity 0.0
    wavespeed 1.0
    is_basename True
    >>> gridfunc1, gridfunc2 = register_gridfunctions(['gridfunc1', 'gridfunc2'], f_infinity=[1.0, 4.0])
    >>> for key, value in glb_gridfcs_dict["gridfunc1"].__dict__.items():
    ...     print(key, value)
    c_type_alias CCTK_REAL
    name gridfunc1
    group EVOL
    rank 0
    f_infinity [1.0, 4.0]
    wavespeed 1.0
    is_basename True
    """
    # Step 1: Convert names to a list if it's not already a list
    if not isinstance(names, list):
        names = [names]

    # Step 2: Process each gridfunction
    sympy_symbol_list: List[sp.Symbol] = []
    Infrastructure = par.parval_from_str("Infrastructure")
    for i, name in enumerate(names):
        if name in glb_gridfcs_dict:
            print(f"Warning: Gridfunction {name} is already registered.")
        else:
            gf: Union[BHaHGridFunction, BaseETGridFunction, CarpetXGridFunction]
            if Infrastructure == "BHaH":
                kwargs_modify = kwargs.copy()
                for param in ["f_infinity", "wavespeed"]:
                    if kwargs.get(param) and isinstance(kwargs.get(param), list):
                        # mypy: Once again bonks out after I've CONFIRMED kwargs.get(param) is not None and is a list!
                        kwargs_modify[param] = kwargs.get(param)[i]  # type: ignore
                gf = BHaHGridFunction(name, **kwargs_modify)
            elif Infrastructure == "BaseET":
                gf = BaseETGridFunction(name, **kwargs)
            elif Infrastructure == "CarpetX":
                gf = CarpetXGridFunction(name, **kwargs)
            else:
                raise ValueError(f"Infrastructure = {Infrastructure} unknown")

            glb_gridfcs_dict[name] = gf
        sympy_symbol_list += [sp.symbols(name, real=True)]

    # Step 3: Return the list of sympy symbols, which can have any number of elements >= 1
    return sympy_symbol_list

def register_gridfunctions_for_single_rank1(
    basename: str, **kwargs: Any
) -> Sequence[sp.Expr]:
    """
    Register gridfunctions for a single rank 1 variable.

    :param basename: Base name of the gridfunction to register.
    :param kwargs: Additional keyword arguments for registration.
    :return: List of SymPy symbols.

    Doctests:
    >>> glb_gridfcs_dict.clear()
    >>> register_gridfunctions_for_single_rank1("betU")
    [betU0, betU1, betU2]

    Now make sure this was stored to glb_gridfcs_dict correctly:
    >>> outstr = ""
    >>> for key, gfobj in glb_gridfcs_dict.items():
    ...    outstr += gfobj.name + " "
    >>> print(outstr[:-1])
    betU0 betU1 betU2
    """
    dimension = kwargs.get("dimension", 3)
    if "dimension" in kwargs:
        del kwargs["dimension"]

    # Step 1: Declare a list of SymPy variables,
    #         where IDX_OBJ_TMP[i] = gf_basename+str(i)
    IDX_OBJ_TMP = ixp.declarerank1(basename, dimension=dimension, **kwargs)

    # Step 2: Register each gridfunction
    gf_list = []
    for i in range(3):
        gf_list += [str(IDX_OBJ_TMP[i])]
    # Manually adjust kwargs so that basename is not checked (we're not passing the basename!),
    # and rank is set to 1.
    kwargs["is_basename"] = False
    kwargs["rank"] = 1
    register_gridfunctions(gf_list, **kwargs)

    # Step 3: Return array of SymPy variables
    return IDX_OBJ_TMP

def register_gridfunctions_for_single_rank2(
    basename: str, symmetry: Optional[str] = None, **kwargs: Any
) -> Sequence[Sequence[sp.Expr]]:
    """
    Register gridfunctions for a single rank 2 variable.

    :param basename: Base name of the gridfunction to register.
    :param symmetry: Optional symmetry for rank 2 gridfunctions.
    :param kwargs: Additional keyword arguments for registration.
    :return: 2D list of SymPy symbols.

    Doctests:
    >>> glb_gridfcs_dict.clear()
    >>> register_gridfunctions_for_single_rank2("gDD", symmetry="sym01")
    [[gDD00, gDD01, gDD02], [gDD01, gDD11, gDD12], [gDD02, gDD12, gDD22]]

    Now make sure this was stored to glb_gridfcs_dict correctly:
    >>> outstr = ""
    >>> for key, gfobj in glb_gridfcs_dict.items():
    ...    outstr += gfobj.name + " "
    >>> print(outstr[:-1])
    gDD00 gDD01 gDD02 gDD11 gDD12 gDD22
    """
    dimension = kwargs.get("dimension", 3)
    if "dimension" in kwargs:
        del kwargs["dimension"]

    # Step 1: Declare a list of lists of SymPy variables,
    #         where IDX_OBJ_TMP[i][j] = gf_basename+str(i)+str(j)
    IDX_OBJ_TMP = ixp.declarerank2(
        basename, symmetry=symmetry, dimension=dimension, **kwargs
    )

    # Step 2: register each gridfunction, being careful not
    #         not to store duplicates due to rank-2 symmetries.
    gf_list: List[str] = []
    for i in range(dimension):
        for j in range(dimension):
            save = True
            for gf in gf_list:
                if gf == str(IDX_OBJ_TMP[i][j]):
                    save = False
            if save:
                gf_list.append(str(IDX_OBJ_TMP[i][j]))

    # Manually adjust kwargs so that basename is not checked (we're not passing the basename!),
    # and rank is set to 2.
    kwargs["is_basename"] = False
    kwargs["rank"] = 2
    register_gridfunctions(gf_list, **kwargs)

    # Step 3: Return array of SymPy variables
    return IDX_OBJ_TMP

# Outside the classes above, only define functions in grid.py that are used by multiple infrastructures
#####################################

if __name__ == "__main__":
    import doctest
    import sys

    results = doctest.testmod()

    if results.failed > 0:
        print(f"Doctest failed: {results.failed} of {results.attempted} test(s)")
        sys.exit(1)
    else:
        print(f"Doctest passed: All {results.attempted} test(s) passed")


from flask import Flask, render_template, redirect, url_for
import datetime
import matplotlib.pyplot as plt
import os

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def home():

    current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")

    left = [1, 2, 3, 4, 5]
    # heights of bars
    height = [10, 24, 36, 40, 5]
    # labels for bars
    tick_label = ['one', 'two', 'three', 'four', 'five']
    # plotting a bar chart
    plt.bar(left,
            height,
            tick_label=tick_label,
            width=0.8,
            color=['red', 'green'])

    # naming the y-axis
    plt.ylabel('y - axis')
    # naming the x-axis
    plt.xlabel('x - axis')
    # plot title
    plt.title('My bar chart!')

    plt.savefig(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             'static/images/plot.png'))

    return render_template('dashboard.html',
                           current_time=current_time,
                           plot='/static/images/plot.png')

if __name__ == '__main__':
    app.run()


import sys
import hashlib
import libnum
import base64

e = 65537
p = 3
q = 5
n = p * q
phi = (p-1) * (q-1)
d = libnum.invmod(e, phi)

file_obj = open(r"input.txt", "r")
message = file_obj.readlines()
file_obj.close()

message = ''.join(message)

encoded_bytes = base64.b64encode(message.encode("utf-8"))
# encoded_str = str(encoded_bytes, 'utf-8')
int_b = int.from_bytes(encoded_bytes, 'big')
print()
print(encoded_bytes)
print()
print(int_b)
c = pow(int_b, e, n)

print()
print(encoded_bytes)
print()
print(c)
print()

dc = pow(c, d, n)

dc = (dc).to_bytes((dc.bit_length() + 7) // 8, byteorder='big')

encoded_str = base64.b64decode(dc)
res = str(encoded_str, 'utf-8')
print(res)


import numpy as np

def do_nms(dets_all, n_class, th_nms):
    '''
    dets: detection results, (n_dets, 6)
    each row of dets: (class_id, class_prob, xmin, ymin, xmax, ymax)
    '''
    cidx_all = []
    for c in range(n_class):
        cidx = np.where(dets_all[:, 0] == c)[0]
        if cidx.size == 0:
            continue
        dets = dets_all[cidx, :]
        sidx = np.argsort(dets[:, 1])[::-1]
        cidx = cidx[sidx]
        dets = dets[sidx, :]
        vidx = _nms(dets, th_nms)
        cidx = cidx[vidx]
        cidx_all.append(cidx)
    try:
        if cidx_all:
            cidx_all = np.hstack((cidx_all))
        nidx = np.setdiff1d(np.arange(dets_all.shape[0]), cidx_all)
    except:
        import ipdb
        ipdb.set_trace()
    dets_all[nidx, 0] = -1
    return dets_all

def _nms(dets, th_nms):
    '''
    '''
    areas = (dets[:, 4] - dets[:, 2]) * (dets[:, 5] - dets[:, 3])
    vmask = np.ones((dets.shape[0],), dtype=int)
    vidx = []
    for i, d in enumerate(dets):
        if vmask[i] == 0:
            continue
        iw = np.minimum(d[4], dets[i:, 4]) - np.maximum(d[2], dets[i:, 2])
        ih = np.minimum(d[5], dets[i:, 5]) - np.maximum(d[3], dets[i:, 3])
        I = np.maximum(iw, 0) * np.maximum(ih, 0)
        iou = I / np.maximum(areas[i:] + areas[i] - I, 1e-08)
        nidx = np.where(iou > th_nms)[0] + i
        vmask[nidx] = 0
        vidx.append(i)
    return vidx


import pandas as pd
from sklearn.svm import OneClassSVM
from sklearn import preprocessing
import matplotlib.pyplot as plt
from numpy import where

def get_normalized_dataframe(df: pd.DataFrame) -> pd.DataFrame:
    df_values = df.values
    standard_scaler = preprocessing.StandardScaler()
    df_values_scaled = standard_scaler.fit_transform(df_values)
    return pd.DataFrame(df_values_scaled)

def detect_anomalies_and_save_to_csv(computing_type: str, check: bool = False, normalization: bool = False) -> None:
  data = pd.read_csv('xd.csv')

  df = data.loc[data['type'] == computing_type]
  df = df.drop('type', axis='columns')

  if normalization:
    df_normalized = get_normalized_dataframe(df)

    model = OneClassSVM(kernel = 'rbf', gamma = 5e-2, nu = 0.05).fit(df_normalized)
    y_pred = model.predict(df_normalized)

  else:
    model = OneClassSVM(kernel = 'rbf', gamma = 1e-9, nu = 0.05).fit(df)
    y_pred = model.predict(df)

  outlier_index_list = where(y_pred == -1)
  outlier_df = df.iloc[outlier_index_list]

  normal_guys_index_list = where(y_pred == 1)
  normal_guys_df = df.iloc[normal_guys_index_list]

  if check:
    df_battery_converted = df[' battery'] * -1
    df_battery_converted_outlier = outlier_df[' battery'] * -1

    plt.scatter(df[' time'], df_battery_converted)
    plt.scatter(outlier_df[' time'], df_battery_converted_outlier, c = 'r')
    plt.xlabel('time [ms]', fontsize=12)
    plt.ylabel('battery usage [mAh]', fontsize=12)
    plt.show()

  normal_guys_df.to_csv(f'SVM_{computing_type}_no_anomalies.csv', index=False)

if __name__ == '__main__':
  detect_anomalies_and_save_to_csv('cloud', check=True, normalization=True)
  detect_anomalies_and_save_to_csv('local', check=True, normalization=True)


#This code works!

from email.mime import base
import requests
import pandas as pd
import json
import pathlib
import csv
import calendar
import datetime
import time as t
from datetime import date, datetime, timedelta
from datetime import datetime, time, timezone
from collections import defaultdict
import urllib
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)

#pd.set_option('max_columns', None)
# Function to get all comments for a specific submission and find the keyword "removed" in the comment body.
# If one comment is found with the keyword, then return the comment.
def mySubmissionHasExplanation(submission_id):
    #Convert submision id from base 36 to base 10 and build the url.
    submission_id_converted = int(submission_id,36)
    # First collect all root comments for the submission - endpoint not working
    url= "https://api.pushshift.io/reddit/comment/search/?q='was removed'|'has been removed'&link_id="+str(submission_id_converted)
    print(f"Url for the comments for each submission: {url}")
    try:
        print("In try")
        comments = requests.get(url)
        print("after requests.get")
        print(f"comments: {comments}")
        # dataframe of all comment_ids
        comments_df = pd.json_normalize(comments.json()['data'])
        length_comments_df = len(comments_df)
        print(f"The lenght of the comment_df: {length_comments_df}")

        for i in range (length_comments_df):
            comment_body = comments_df.loc[i, "body"]
            print(f"Comment body: {comment_body}")
            comment_author = comments_df.loc[i, "author"]
            comment_created_utc = comments_df.loc[i, "created_utc"]
            if ("removed" in comment_body): #or ("was removed" in body) or ("has been removed" in body):
                return comment_body, comment_author, comment_created_utc

    except:
        print("The endpoint could not be reached.")
    return "None", "None", "None"

# Get current working directory
cwd_path = pathlib.Path.cwd()

# Create a new folder to write any files related to reddit_explanations_covid_pushshift
reddit_explanations_pushshift = cwd_path/"reddit_explanations_covid_pushshift"
reddit_explanations_pushshift.mkdir(exist_ok=True)

submissions_file = reddit_explanations_pushshift/"submissions_file.csv"
# Create a file called removed_submissions.csv to write all removed submissions from July 1st to August 31st.
removed_submissions_file = reddit_explanations_pushshift/"removed_submissions.csv"
# removed_submissions_comments_file = reddit_explanations_pushshift/"removed_submissions_comments.csv"

submissions_file.touch()
removed_submissions_file.touch()
# removed_submissions_comments_file.touch()

# List for all submissions id
submissions_id_list = []
comments_list = []
#keys_for_comments_dict = ["submission_id", "comment_body", "comment_author", "comment_created_utc"]
comments_dict = defaultdict(list)
# Get 500 submissions per day with the keywords covid or vaccine between the dates July 1st and August 31st
# Get a list of all the dates between July 1st and August 31st
num_days = 1
# Create base date
base = datetime(2022, 12, 29)
date_list = []
submissions_df = pd.DataFrame()
# Create a list of dates starting on July 1st
for x in range(num_days):
    date_list.append(base + timedelta(days=x))

# Loop through date_list and get begin and end datetimes for each day in utc. Create url to call pushshift api and
# get 500 removed submissions by moderator for each day and add to submissions dataframe.
for dt in date_list:
    begintime_date = datetime.combine(dt, time.min)
    begintime_date = datetime.combine(dt, time.min)# time.min will give me 12:00 am
    print("begintime_date",begintime_date)
    endtime_date = datetime.combine(dt, time.max) # time.max will give me 11:59 pm
    begintime_date_utc = begintime_date.timestamp()
    print("begintime_date_utc",begintime_date_utc)
    endtime_date_utc = endtime_date.timestamp()
    url = "https://api.pushshift.io/reddit/search/submission/?" #This is the pushshift endpoint for getting the submissions.
    params = {"since": int(begintime_date_utc), "until": int(endtime_date_utc), "size": "200", "removed_by_category": "moderator"}
    url = url + urllib.parse.urlencode(params) # Forming the url
    print(url)
    try:
        data= requests.get(url)
        # Add the submissions received from Reddit to a temporary dataframe and add to submissions dataframe.
        # Keep adding to submissions_df each time you loop.
        temp_submissions_df = pd.json_normalize(data.json()['data'])
        #submissions_df = submissions_df.append(temp_submissions_df, ignore_index=True)
        submissions_df = pd.concat([submissions_df,temp_submissions_df])
    except:
        print("The endpoint could not be reached.")

    t.sleep(10)
# Write all the submissions with at least 1 comment containing the keyword "removed" to a file
if not submissions_df.empty:
    submissions_df_with_comments = submissions_df[(submissions_df["num_comments"] > 0)]
    data_top = submissions_df_with_comments.head()
    #print("data_top",data_top)
    list(submissions_df_with_comments.columns)
    #for col_name in submissions_df_with_comments.columns:
        #print(f"Column name: {col_name}")
    submissions_df_with_comments.to_csv(submissions_file)
    for i in range(len(submissions_df_with_comments)):
        try:
            submission_id = submissions_df_with_comments.iloc[i, 28]
            print(f"The submission_id is: {submission_id}")
            # For each submission, check if at least 1 comment contains the word "removed". If True, then add the
            # submission_id to the submissions_id_list.
            comment_body, comment_author, comment_created_utc = mySubmissionHasExplanation(submission_id)
            print(f"Comment author: {comment_author}")
            print(f"Comment body: {comment_body}")
            if comment_body == "None":
                print("This submission has no comments with the word 'removed'.")
            else:
                # Add submission with comment with the word "removed" to the submissions_id_list
                submissions_id_list.append(submission_id)
                #comments_list.extend([submission_id, comment_body, comment_author, comment_created_utc])

                comments_dict["submission_id"].append(submission_id)
                comments_dict["comment_body"].append(comment_body)
                comments_dict["comment_author"].append(comment_author)
                comments_dict["comment_created_utc"].append(comment_created_utc)
        except:
            print("Could not get submission_id.")
        t.sleep(7)
    if comments_dict:
        comments_dict_df = pd.DataFrame(comments_dict)
        print(comments_dict_df.shape)
        # Merge submissions_df_with_comments and comments_list_df with id as left key and submission_id as the right key

        submissions_df_to_write = submissions_df_with_comments[submissions_df_with_comments["id"].isin(submissions_id_list)]
        print(submissions_df_to_write.shape)

        submissions_df_to_write_merged = pd.merge(submissions_df_to_write, comments_dict_df, left_on='id', right_on='submission_id')

        header = ["id", "url", "selftext", "author_fullname", "created_utc", "score", "subreddit", "permalink", 'comment_body', 'comment_author', 'comment_created_utc']
        submissions_df_to_write_merged.to_csv(removed_submissions_file, index=False, columns=header)

    # Sources:

    # https://levelup.gitconnected.com/easily-obtain-reddit-data-with-python-8ce0c576540f

    # https://reddit-api.readthedocs.io/en/latest/#searching-submissions

    # https://github.com/pushshift/api


from flask import current_app as app
from socketio import Namespace
from urllib.parse import urlsplit, parse_qsl

import datetime
import pprint

class ChatUtils(object):
  def __init__(self, sio, *args, **kwargs):
    self.sio = sio
    self.args = args
    self.kwargs = kwargs

  def get_session_user(self, _sid, attr=None, attrs=None):
    session = self.sio.get_session(_sid)

    user = session.get("user", self.dummy_user.copy())

    if isinstance( attr, str ):
      return user[attr]

    elif isinstance( attrs, list ):
      temp = dict()
      for key in attrs:
        temp[key] = user[key]
      return temp

    return user.copy()

  def set_session_user(self, _sid, **kwargs):
    session = self.sio.get_session(_sid)

    user = session.get("user", None)
    if user is None:
      user = self.dummy_user.copy()
    user.update( kwargs )

    session.update({ "user": user })

  def parse_query_string_to_dict(self, query_string):
    return dict(parse_qsl(urlsplit(query_string).path))

class ChatNamespace(Namespace, ChatUtils):
  def __init__(self, sio, namespace, *args, **kwargs):
    super(Namespace, self).__init__(namespace)

    self.sio = sio
    self.args = args
    self.kwargs = kwargs

    self.logger = app.logger

    self.default_room = "lobby"
    self.dummy_user = {
      "sid": None,
      "name": "Unknown",
      "room": "lobby",
      "auth": "normal",
      "loggedIn": None
    }

  def on_connect(self, sid, env):
    remote_addr = env["REMOTE_ADDR"]
    query_string = self.parse_query_string_to_dict(env["QUERY_STRING"])

    self.logger.info(f'[CONNECT] { remote_addr } { sid }')
    self.logger.info(f'[CONNECT][QUERY_STRING] { query_string }')

    self.set_session_user(sid, sid=sid, name=query_string.get("username", "Unknown"))
    self.enter_room(sid, self.default_room)

    emit_data = {
      "user": self.get_session_user(sid),
      "room": self.default_room
    }

    self.emit("room_hello", emit_data, room=self.default_room)
    self.emit("connected", emit_data, room=self.default_room)

  def on_disconnect(self, sid):
    self.logger.info(f"[DISCONNECT] { sid }")

  def on_set_username(self, sid, username):
    user = self.get_session_user(sid)

    self.set_session_user(sid, name=username)

    self.logger.info( f'[SET_USERNAME] { user["name"] } -> { username }')

    return (True, )

  def on_send_message(self, sid, send_data):
    user = self.get_session_user(sid)

    self.logger.info(f'[SEND_MESSAGE][room={ user["room"] }] { user["name"] }: { send_data["message"] }')

    emit_data = send_data.copy()
    emit_data.update({
      "user": user,
    })

    self.emit("receive_message", emit_data, room=send_data["room"])

  def on_enter_room(self, sid, enter_room):
    prev_room = self.get_session_user(sid, attr="room")

    if prev_room != enter_room:
      leave_emit_data = {
        "user": self.get_session_user(sid),
        "room": prev_room
      }
      self.emit("room_bye", leave_emit_data, room=prev_room)
      self.leave_room(sid, prev_room)

      self.logger.info(f'[LEAVE_ROOM][room={ prev_room }]')

    self.enter_room(sid, enter_room)
    self.set_session_user(sid, room=enter_room)

    enter_emit_data = {
      "user": self.get_session_user(sid),
      "room": enter_room
    }

    self.logger.info(f'[ENTER_ROOM][room={ enter_room }]')
    self.logger.info(f'[ENTER_ROOM][ROOMS] { self.rooms(sid) }')

    self.emit("room_hello", enter_emit_data, room=enter_room)

    return (True, )

  def on_leave_room(self, sid, leave_room):
    self.leave_room(sid, leave_room)
    self.set_session_user(sid, room=self.default_room)

    emit_data = {
      "user": self.get_session_user(sid),
      "room": leave_room
    }

    self.emit("room_bye", emit_data, room=leave_room)

    return (True, )


import matplotlib.pyplot as plt
import numpy as np

from paddlemix.appflow import Appflow
from ppdiffusers.utils import load_image

def show_mask(mask, ax, random_color=False):
    if random_color:
        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
    else:
        color = np.array([30 / 255, 144 / 255, 255 / 255, 0.6])
    h, w = mask.shape[-2:]
    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
    ax.imshow(mask_image)

def show_box(box, ax, label):
    x0, y0 = box[0], box[1]
    w, h = box[2] - box[0], box[3] - box[1]
    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor="green", facecolor=(0, 0, 0, 0), lw=2))
    ax.text(x0, y0, label)

task = Appflow(
    app="auto_label",
    models=["paddlemix/blip2-caption-opt2.7b", "GroundingDino/groundingdino-swint-ogc", "Sam/SamVitH-1024"],
)
url = "https://paddlenlp.bj.bcebos.com/models/community/CompVis/stable-diffusion-v1-4/overture-creations.png"
image_pil = load_image(url)
blip2_prompt = "describe the image"
result = task(image=image_pil, blip2_prompt=blip2_prompt)

plt.figure(figsize=(10, 10))
plt.imshow(result["image"])
for mask in result["seg_masks"]:
    show_mask(mask.cpu().numpy(), plt.gca(), random_color=True)
    for box, label in zip(result["boxes"], result["labels"]):
        show_box(box, plt.gca(), label)

plt.axis("off")
plt.savefig(
    "mask_pred.jpg",
    bbox_inches="tight",
    dpi=300,
    pad_inches=0.0,
)


import boto3, json

dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('cmup_users')

def lambda_handler(event, context):

    response = table.get_item(
        Key={
            'id': event['pathParameters']['user']
        }
    )

    user = response['Item']

    return {
        'statusCode': 200,
        'body': json.dumps(user)
    }


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse

from cats_dogs.dnn_api import ModelWrapper

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train cat/doc classification model.')
    parser.add_argument('model_path', type=str,
                        help='path to save trained model')
    parser.add_argument('train_dir', type=str,
                        help='path to directory with train images')
    parser.add_argument('val_dir', type=str,
                        help='path to directory with validation images')
    parser.add_argument('n_epochs', type=int,
                        help='number of epochs for training')
    parser.add_argument('--load', type=bool, default=True,
                        help='load pre-trained model if True, initialize new if False. True by default')
    args = parser.parse_args()

    if args.load:
        model = ModelWrapper.load(args.model_path)
    else:
        model = ModelWrapper.new()
    model.train(args.train_dir,
                args.val_dir,
                n_epochs=args.n_epochs)
    model.save(args.model_path)


# -*- coding: utf-8 -*-
"""
ofood商户排行数据
"""
import airflow
from airflow.operators.python_operator import PythonOperator
from airflow.operators.impala_plugin import ImpalaOperator
from datetime import datetime, timedelta
from utils.connection_helper import get_hive_cursor, get_db_conn, get_db_conf
from utils.validate_metrics_utils import *
from airflow.sensors.s3_prefix_sensor import S3PrefixSensor
from airflow.operators.bash_operator import BashOperator
from airflow.sensors import UFileSensor
from plugins.comwx import ComwxApi
from airflow.sensors import OssSensor
import time
import logging

args = {
    'owner': 'wuduo',
    'start_date': datetime(2019, 8, 18),
    'depends_on_past': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=5),
    'email': ['bigdata_dw@opay-inc.com'],
    'email_on_failure': True,
    'email_on_retry': False,
}

dag = airflow.DAG(
    'app_ofood_merchant_offline_tag_d',
    schedule_interval="00 04 * * *",
    concurrency=5,
    max_active_runs=1,
    default_args=args
)

"""
依赖采集完成
"""
dependence_ods_sqoop_base_jh_order_df = OssSensor(
    task_id='dependence_ods_sqoop_base_jh_order_df',
    bucket_key='{hdfs_path_str}/dt={pt}'.format(
        hdfs_path_str="ofood_dw_sqoop/food_operapay_co/jh_order",
        pt='{{ ds }}'
    ),
    bucket_name='opay-datalake',
    poke_interval=60,
    dag=dag
)

dependence_ods_sqoop_base_jh_order_log_df = OssSensor(
    task_id='dependence_ods_sqoop_base_jh_order_log_df',
    bucket_key='{hdfs_path_str}/dt={pt}'.format(
        hdfs_path_str="ofood_dw_sqoop/food_operapay_co/jh_order_log",
        pt='{{ ds }}'
    ),
    bucket_name='opay-datalake',
    poke_interval=60,
    dag=dag
)

dependence_ods_sqoop_base_jh_waimai_comment_df = OssSensor(
    task_id='dependence_ods_sqoop_base_jh_waimai_comment_df',
    bucket_key='{hdfs_path_str}/dt={pt}'.format(
        hdfs_path_str="ofood_dw_sqoop/food_operapay_co/jh_waimai_comment",
        pt='{{ ds }}'
    ),
    bucket_name='opay-datalake',
    poke_interval=60,
    dag=dag
)

dependence_ods_sqoop_base_jh_order_time_df = OssSensor(
    task_id='dependence_ods_sqoop_base_jh_order_time_df',
    bucket_key='{hdfs_path_str}/dt={pt}'.format(
        hdfs_path_str="ofood_dw_sqoop/food_operapay_co/jh_order_time",
        pt='{{ ds }}'
    ),
    bucket_name='opay-datalake',
    poke_interval=60,
    dag=dag
)

"""
end
"""

sleep_time = BashOperator(
    task_id='sleep_id',
    depends_on_past=False,
    bash_command='sleep 120',
    dag=dag
)

def get_data_from_impala(**op_kwargs):
    ds = op_kwargs.get('ds', time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400)))
    sql = '''
        WITH
        --商家取消单
        merchant_data as
        (
            select
                a1.shop_id,
                count(distinct a1.order_id) cancel_order
            from (select
                    shop_id,
                    order_id
                from ofood_dw_ods.ods_sqoop_base_jh_order_df
                where dt='{pt}' and
                    order_status='-2' and
                    from_unixtime(dateline,'yyyy-MM-dd') between date_sub('{pt}', 6) and '{pt}'
                ) as a1
            join (select
                    order_id
                from ofood_dw_ods.ods_sqoop_base_jh_order_log_df
                where dt='{pt}' and
                    log like '%Merchant cancelling order%'
                ) as a2
            on a1.order_id=a2.order_id
            group by a1.shop_id
        ),
        --评价单小于等于2
        score_data as (
            select
                a1.shop_id,
                count(distinct a1.order_id) as scorelttwo
            from (select
                    shop_id,
                    order_id
                from ofood_dw_ods.ods_sqoop_base_jh_order_df
                where dt='{pt}' and
                    order_status='8' and
                    from_unixtime(dateline,'yyyy-MM-dd') between date_sub('{pt}', 6) and '{pt}'
                ) as a1
            join (select
                    order_id
                from ofood_dw_ods.ods_sqoop_base_jh_waimai_comment_df
                where dt='{pt}' and
                    score_peisong<=2
                ) as a2
            on a1.order_id = a2.order_id
            group by a1.shop_id
        ),
        --大于90分的单子
        order_data as (
            select
                s.shop_id,
                count(distinct s.order_id) as ordersgt90
            from (select
                    a.shop_id,
                    a.order_id,
                    datediff(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'),from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss'))*24*60+
                    (hour(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'))-hour(from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss')))*60+(minute(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'))-minute(from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss'))) as tm
                from (select
                        shop_id,
                        order_id,
                        pay_time
                    from ofood_dw_ods.ods_sqoop_base_jh_order_df
                    where dt='{pt}' and
                        order_status = '8' and
                        from_unixtime(dateline,'yyyy-MM-dd')  between date_sub('{pt}', 6) and '{pt}'
                    ) as a
                join (select
                        order_id,
                        order_compltet_time
                    from ofood_dw_ods.ods_sqoop_base_jh_order_time_df
                    where dt='{pt}'
                    ) as b
                on a.order_id=b.order_id
                group by a.shop_id, a.order_id, datediff(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'),from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss'))*24*60+
                    (hour(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'))-hour(from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss')))*60+(minute(from_unixtime(b.order_compltet_time,'yyyy-MM-dd HH:mm:ss'))-minute(from_unixtime(a.pay_time,'yyyy-MM-dd HH:mm:ss')))
                ) s
            where s.tm>90
            group by s.shop_id
        ),
        --近7天下单量
        order_7day as (
            select
                shop_id,
                count(distinct order_id) as orders
            from ofood_dw_ods.ods_sqoop_base_jh_order_df
            where dt='{pt}' and
                from_unixtime(dateline,'yyyy-MM-dd')  between date_sub('{pt}', 6) and '{pt}'
            group by shop_id
        ),
        --近30天完单量
        order_30day as (
            select
                shop_id,
                count(distinct order_id) as orderfinish
            from ofood_dw_ods.ods_sqoop_base_jh_order_df
            where dt='{pt}' and
                from_unixtime(dateline,'yyyy-MM-dd')  between date_sub('{pt}', 29) and '{pt}' and
                order_status = '8'
            group by shop_id
        )

        --结果集
        select
            order_7day.shop_id,
            nvl(merchant_data.cancel_order, 0),
            nvl(score_data.scorelttwo, 0),
            nvl(order_data.ordersgt90, 0),
            order_7day.orders,
            nvl(order_30day.orderfinish, 0),
            unix_timestamp('{pt}', 'yyyy-MM-dd')
        from order_7day
        left join merchant_data on order_7day.shop_id = merchant_data.shop_id
        left join score_data on order_7day.shop_id = score_data.shop_id
        left join order_data on order_7day.shop_id = order_data.shop_id
        left join order_30day on order_7day.shop_id = order_30day.shop_id
    '''.format(
        pt=ds
    )
    logging.info(sql)
    hive_cursor = get_hive_cursor()
    hive_cursor.execute(sql)
    result = hive_cursor.fetchall()

    mysql_conn = get_db_conn('mysql_bi')
    mcursor = mysql_conn.cursor()
    __data_to_mysql(mcursor, result,
                    ['shop_id', 'last7_cancel_order', 'last7_low_scores_2_order', 'last7_forward_time_90_order',
                     'last7_request_order', 'last30_finish_order', 'update_time'],
                    '''
                        last7_cancel_order=values(last7_cancel_order),
                        last7_low_scores_2_order=values(last7_low_scores_2_order),
                        last7_forward_time_90_order=values(last7_forward_time_90_order),
                        last7_request_order=values(last7_request_order),
                        last30_finish_order=values(last30_finish_order),
                        update_time=values(update_time)
                    '''
                    )

def __data_to_mysql(conn, data, column, update=''):
    isql = 'insert into bi.ofood_merchant_offline_tag ({})'.format(','.join(column))
    esql = '{0} values {1} on duplicate key update {2}'
    sval = ''
    cnt = 0
    try:
        for (shop_id, cancel_order, scorelttwo, ordersgt90, orders, orderfinish, upt) in data:

            row = [shop_id, cancel_order, scorelttwo, ordersgt90, orders, orderfinish, upt]
            if sval == '':
                sval = '(\'{}\')'.format('\',\''.join([str(x) for x in row]))
            else:
                sval += ',(\'{}\')'.format('\',\''.join([str(x) for x in row]))
            cnt += 1
            if cnt >= 1000:
                logging.info(esql.format(isql, sval, update))
                conn.execute(esql.format(isql, sval, update))
                cnt = 0
                sval = ''

        if cnt > 0 and sval != '':
            logging.info(esql.format(isql, sval, update))
            conn.execute(esql.format(isql, sval, update))
    except BaseException as e:
        logging.info(e)
        return

get_data_from_impala_task = PythonOperator(
    task_id='get_data_from_impala_task',
    python_callable=get_data_from_impala,
    provide_context=True,
    dag=dag
)

comwx = ComwxApi('wwd26d45f97ea74ad2', 'BLE_v25zCmnZaFUgum93j3zVBDK-DjtRkLisI_Wns4g', '1000011')

def check_ds_data(**op_kwargs):
    ds = op_kwargs.get('ds', time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400)))
    sql = '''
        select count(1) as cnt from bi.ofood_merchant_offline_tag where from_unixtime(update_time,'%Y-%m-%d')='{pt}'
    '''.format(pt=ds)
    mysql_conn = get_db_conn('mysql_bi')
    mcursor = mysql_conn.cursor()
    mcursor.execute(sql)
    res = mcursor.fetchall()
    logging.info(sql)
    logging.info(res)
    logging.info(isinstance(res, tuple))
    logging.info(len(res))
    logging.info(res[0])
    if res is None or not isinstance(res, tuple) or len(res) <= 0:
        comwx.postAppMessage('ofood商家订单指标缺少{}数据, 请及时排查'.format(ds), '271')
    else:
        (cnt,) = res[0]
        logging.info(cnt)
        if cnt <= 0:
            comwx.postAppMessage('ofood商家订单指标缺少{}数据, 请及时排查'.format(ds), '271')

check_data = PythonOperator(
    task_id='check_data',
    python_callable=check_ds_data,
    provide_context=True,
    dag=dag
)

dependence_ods_sqoop_base_jh_order_df >> sleep_time
dependence_ods_sqoop_base_jh_order_log_df >> sleep_time
dependence_ods_sqoop_base_jh_waimai_comment_df >> sleep_time
dependence_ods_sqoop_base_jh_order_time_df >> sleep_time

sleep_time >> get_data_from_impala_task >> check_data


import re
from collections import Counter
from mpi4py import MPI

import nltk
from razdel import sentenize
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

def tokenize_by_sent(text):
    return [_.text for _ in (sentenize(text))]

def validation(word):
    match = re.match("^[А-Яа-яё]+$", word)
    return bool(match)

def divide_chunks(l, n):
    return [l[i::n] for i in range(n)]

nltk.download('punkt')
nltk.download('stopwords')
ru_stop_words = set(stopwords.words('russian'))

comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()

with open('data/war_and_peace.txt') as fin:
    input_text = ' '.join(fin.readlines())

if rank == 0:
    sentences = tokenize_by_sent(input_text)
    segments = divide_chunks(sentences, size)

else:
    segments = None

segment = comm.scatter(segments, root=0)

tokens = word_tokenize(' '.join(segment), language='russian')
tokens = [word.lower() for word in tokens if validation(word)]
filtered_words = [word.lower() for word in tokens if word not in ru_stop_words]
word_counts = Counter(filtered_words)

word_counts = comm.gather(word_counts, root=0)

if rank == 0:
    top_words = sum(word_counts, Counter()).most_common(10)
    print("Top word")
    for i, word in enumerate(top_words):
        print("{0} - {1}:{2}".format(i, word[0], word[1]))


import pytest

from context import necklaces_generation
from necklaces_generation import *

from equivalent_sequences import find_rotation_and_reversal_orbit

def test_find_necklace():
    sequence = [1, 0, 0, 1]
    neckalace = [0, 0, 1, 1]
    assert find_necklace(sequence) == neckalace

def test_equal_necklaces_true():
    assert equal_necklaces([1, 0, 1, 1], [1, 1, 1, 0])

def test_equal_necklaces_false():
    assert not equal_necklaces([1, 0, 1, 1], [1, 0, 1, 0])

@pytest.mark.parametrize(
    "seq, seq_successor",
    [
        ([0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 1]),
        ([0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 1, 0]),
        ([0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 1, 1]),
        ([0, 0, 0, 0, 1, 1], [0, 0, 0, 1, 0, 0]),
        ([0, 0, 0, 1, 0, 0], [0, 0, 0, 1, 0, 1]),
        ([0, 0, 0, 1, 0, 1], [0, 0, 0, 1, 1, 0]),
        ([0, 0, 0, 1, 1, 0], [0, 0, 0, 1, 1, 1]),
        ([0, 0, 0, 1, 1, 1], [0, 0, 1, 0, 0, 1]),
        ([1, 1, 1, 1, 1, 1], None),
    ],
)
def test_successor(seq, seq_successor):
    assert next_prenecklace(seq) == seq_successor

@pytest.mark.parametrize(
    "sequence, index",
    [
        ([0, 1, 1, 0, 1], 3),
        ([0, 1, 1, 1, 1], 0),
        ([1, 1, 1, 1, 0], 4),
        ([1, 1, 1, 1, 1], None),
    ],
)
def test_find_largest_index_equal_to_zero(sequence, index):
    assert find_largest_index_equal_to_zero(sequence) == index

def test_fkm_algorithm():
    start = [0] * 6
    end = [1] * 6
    generated_necklaces = [seq for seq in fkm_algorithm(start, end)]
    necklaces = [
        [0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 1],
        [0, 0, 0, 0, 1, 1],
        [0, 0, 0, 1, 0, 1],
        [0, 0, 0, 1, 1, 1],
        [0, 0, 1, 0, 0, 1],
        [0, 0, 1, 0, 1, 1],
        [0, 0, 1, 1, 0, 1],
        [0, 0, 1, 1, 1, 1],
        [0, 1, 0, 1, 0, 1],
        [0, 1, 0, 1, 1, 1],
        [0, 1, 1, 0, 1, 1],
        [0, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1],
    ]
    assert generated_necklaces == necklaces

def test_fkm_algorithm_returns_none():
    assert [seq for seq in fkm_algorithm([0, 0, 1], [0, 0])] == []

def test_seq_necklaces_of_half_density():
    expected_necklaces = [
        [0, 0, 0, 0, 1, 1, 1],
        [0, 0, 0, 1, 0, 1, 1],
        [0, 0, 0, 1, 1, 0, 1],
        [0, 0, 1, 0, 0, 1, 1],
        [0, 0, 1, 0, 1, 0, 1],
    ]

    necklaces = [seq for seq in seq_necklaces_of_half_density(7)]
    assert expected_necklaces == necklaces

def test_find_bracelet():
    expected_bracelet = [0, 0, 1, 0, 1, 1, 1]
    assert find_bracelet([1, 1, 0, 1, 0, 0, 1]) == expected_bracelet

def test_find_bracelet_from_necklace():
    expected_bracelet = [0, 0, 1, 0, 1, 1, 1]
    assert find_bracelet_from_necklace([0, 0, 1, 1, 1, 0, 1]) == expected_bracelet

def test_seq_bracelets_of_half_density():
    expected_necklaces = [
        [0, 0, 0, 0, 1, 1, 1],
        [0, 0, 0, 1, 0, 1, 1],
        [0, 0, 1, 0, 0, 1, 1],
        [0, 0, 1, 0, 1, 0, 1],
    ]

    bracelets = [seq for seq in seq_bracelets_of_half_density(7)]
    assert expected_necklaces == bracelets

def test_seq_module_rotations_and_reversal():
    n = 11
    sequences = [tuple(seq) for seq in seq_bracelets(seq_binary(n))]
    seq_to_orbit = {
        seq: find_rotation_and_reversal_orbit(list(seq)) for seq in sequences
    }

    for seq_a in sequences:
        for seq_b in sequences:
            if seq_a != seq_b:
                assert seq_to_orbit[seq_a] != seq_to_orbit[seq_b]

def test_find_charm_bracelet():
    sequence = [5, 3, 1, 4, 2]
    charm_bracelet = [1, 2, 3, 4, 5]
    assert find_charm_bracelet(sequence) == charm_bracelet

def test_is_charm_bracelet_returns_false_on_non_charm_bracelets():
    sequence = [5, 3, 1, 4, 2]
    assert not is_charm_bracelet(sequence)

def test_is_charm_bracelet_return_true_on_charm_bracelet():
    sequence = [1, 2, 3, 4, 5]
    assert is_charm_bracelet(sequence)

def test_filter_by_charm_bracelet():
    n = 15
    for sequence in filter_by_charm_bracelet(seq_necklaces_of_half_density(n)):
        assert is_charm_bracelet(sequence)


from bs4 import BeautifulSoup
import asyncio
import aiohttp
from config import *
import re
from mysql_script import MySQLi
import datetime

async def get_page_data(session, page, urls_database_dict):
    """
    This is main task
    """
    regex = re.compile('.*cart track-block.*')
    response = await session.get(url=page, headers=headers)
    # print(page)
    if response.status == 200 and '/katalog/' in page:
        soup = BeautifulSoup(await response.text(), 'lxml')
        products_count_real = len(soup.findAll("div", {"class": regex}))  # Количество товаров на странице пагинации
        products_count_html_counter = soup.find("span", {
            "class": 'how_much_founded'})  # Количество товаров, которое указано рядом со страницей сортировки
        products_count_html_counter = products_count_html_counter.text.replace(',', '')
        print(f"Количество товаров на странице пагинации: {products_count_real}\n"
              f"Количество товаров в категории: {products_count_html_counter}\n"
              f"ID DATABASE: {urls_database_dict[page]}\n"
              f"URL: {page}")
        date_now = datetime.datetime.now().date()
        db.commit("INSERT INTO count_products (date, id_url, count_prod, count_html_prod, url) VALUES (%s, %s, %s, %s, %s)"
                  , date_now, urls_database_dict[page], products_count_real, products_count_html_counter, page)

async def gather_data(urls_dabase_dict):
    """
    This is tasks list
    """
    tasks = []
    async with aiohttp.ClientSession() as session:
        for url_for_parse in urls_dabase_dict:
            task = asyncio.create_task(get_page_data(session, url_for_parse, urls_dabase_dict))
            tasks.append(task)

        await asyncio.gather(*tasks)

# def main():
#     asyncio.run(gather_data())

if __name__ == '__main__':
    db = MySQLi(db_host, db_user, db_password, db_name)
    distinct_urls_for_parse = db.fetch("SELECT DISTINCT url, id_url FROM metrika_urls")
    distinct_urls_for_parse = distinct_urls_for_parse['rows']
    urls_dict = {}
    for url in distinct_urls_for_parse:
        # print(url)
        url_data_base = url[0]
        id_url_dabase = url[1]
        urls_dict[url_data_base] = id_url_dabase
    asyncio.run(gather_data(urls_dict))


#!/usr/bin/python3
#Author: Joseph Sevigny

from Bio import SeqIO
import os, gzip

class mess_with_font:
            ENDC = '\033[0m'
            BOLD = '\033[1m'
            UNDERLINE = '\033[4m'
            Red = '\033[91m'
            Green = '\033[92m'
            Blue = '\033[94m'
            Yellow = '\033[93m'
            Grey = '\033[90m'
            Default = '\033[99m'

def do_subprocess(command_list):
    '''performs blasts and returns the results'''
    sp = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    #blast = sp.communicate()
    #blast = str(blast)
    standard_out = str(sp.communicate()[0].decode('ascii'))
    print (command_list)
    return (standard_out)

def extract_file_names(search_term, path_to_dir):
    '''extracts a set of file names that contain a search term and returns it as a list'''
    filenames = []
    file_list = os.listdir(path_to_dir)
    for file_name in file_list:
        if search_term in file_name:
           filenames.append(file_name)
    return filenames

def rev_comp(seq):
    """Reverses, complements and returns sequence"""
    rev_seq = seq[::-1]
    compliment_dict = {'A':'T', 'T':'A', 'C':'G', 'G':'C'}
    rev_comp_seq = ''
    for nuc in rev_seq:
        if nuc in ['A', 'T', 'G', 'C']:
            rev_comp_seq += compliment_dict[nuc]
    return rev_comp_seq

def extract_region(fasta, header_search, start, stop, output_name, direction, header_addition):
    """ PURPOSE: Extracts a specified region from a fasta using header_search, coordinates, and direction(+-),
                        and writes to append able output
        USAGE: extract_region(fasta, header_search, start, stop, output_name, direction, header_addition)"""
    with open(output_name, "a") as my_fasta:
        for seq_record in SeqIO.parse(fasta, "fasta"):
            if header_search in str(seq_record.id):
                my_seq = (str(seq_record.seq[start-1:stop]))
                my_fasta.write(">" + header_addition + "_" + str(len(my_seq)) + "_"+ str(seq_record.id) + "\n")
                if direction == "-":
                    rev_seq = rev_comp(my_seq)
                    my_fasta.write(rev_seq + "\n")
                else:
                    my_fasta.write(my_seq + "\n")

def parse_gff(gff_line):
    '''input a line from a gff and return the important attributes'''
    try:
        header, program, gene_identity, start, stop, tmp, direction, product = gff_line.split("\t")
    except ValueError:
        header, program, gene_identity, start, stop, tmp, direction, tmp2, product = gff_line.split("\t")
    return header, gene_identity, start, stop, direction, product

def match_fasta_and_gff(fastas,gffs):
    fa_gff_dict = {}
    for fa in fastas:
        for gff in gffs:
            if gff.endswith(".gz"):
                tmp_gff = gff.replace(".gff.gz", '')
            else:
                tmp_gff = gff.replace(".gff", '') # just make sure .gz is not in the line
            tmp_gff = tmp_gff.replace("_", ""); tmp_gff = tmp_gff.replace("-", "") #quality control.
            fatmp = fa.replace("_", ""); fatmp = fatmp.replace("-", "") # some more
            if tmp_gff in fatmp:
                fa_gff_dict[fa] = gff #fills the dictionary with fasta and matching gff

    return (fa_gff_dict)

def read_fasta(file_name):
    """reads a fasta file and returns a dictionary with headers as keys and sequences as values"""
    cfo = open(file_name, 'r')
    fasta_dict = {}
    for line in cfo.readlines():
        if line[0] == ">":
            header = line
        else:
            if header not in fasta_dict:
                fasta_dict[header] = line.rstrip()
            else:
                fasta_dict[header] += line.rstrip()
    return fasta_dict


from os import uname
from app import app, db
from app.models import Chat
from flask import render_template, request, redirect
from datetime import datetime

@app.route('/', methods=['GET'])
def index():
    ''' creates a new database file if it does not exist '''
    db.create_all()
    db.session.commit()
    return render_template('index.html', title='Chat APP')

@app.route('/<room>', methods=['GET'])
def room(room):
    return render_template('index.html', title='Chat APP')

@app.route('/api/chat/<room>', methods=['GET', 'POST'])
def room_manage(room):
    if request.method == 'GET':
        res = ''
        if not Chat.query.filter_by(room=room).all():
            return f'No Messages\n\n{uname()}'

        for row in Chat.query.filter_by(room=room).all():
            res += f'[{row.created_at.strftime("%Y-%m-%d %H:%M:%S")}] {row.username}: {row.msg}\n'
        return f'{res}\n{uname()}'

    elif request.method == 'POST':
        new_msg = Chat(
            username=request.form['username'],
            msg=request.form['msg'],
            room=room
        )
        db.session.add(new_msg)
        try:
            db.session.commit()
        except:
            db.session.rollback()

    return render_template('index.html')


import socket
import json
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(('127.0.0.1',1234))

def getJsonFromUrl(urlString):
    #print(urlString)
    response = urllib.request.urlopen(urlString).read().decode()
    responseJson = json.loads(response)
    return responseJson

while 1:
    data, addr = sock.recvfrom(0xffff)
    print("server is received data")
    comments = json.loads(data)
    print(json.dumps(comments, indent=2, sort_keys=True))
    for comment in comments["data"]:
        print("message : " + comment["message"])
        print("writer : " + comment["writer"])


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse
from .models import Post,Category
import markdown
import re
from markdown.extensions.toc import TocExtension
from django.utils.text import slugify

# request是django为我们封装好的http请求，他是HttpRequest的一个实例

def index(request):
   # # 返回一个HttpResponse的一个实例
   # return HttpResponse('欢迎访问我的博客首页！')

   # render根据我们传入的参数构造HttpResponse实例
   # -created_time 中的'-'表示用逆序排序
   post_list = Post.objects.all().order_by('-created_time')
   return render(request, 'blog/index.html', context={'post_list':post_list
   })

def detail(request, pk):
   post = get_object_or_404(Post, pk=pk)
   # post.body = markdown.markdown(post.body,extensions=[
   #    'markdown.extensions.extra',
   #    'markdown.extensions.codehilite',
   #    'markdown.extensions.toc'
   # ])     直接实例化markdown文本
   md = markdown.Markdown(extensions=[
      'markdown.extensions.extra',
      'markdown.extensions.codehilite',
      'markdown.extensions.toc',
       TocExtension(slugify=slugify)])   # 实例化一个对象
   post.body = md.convert(post.body)  # 将文本解析为html，且生成了该文本的目录
   m = re.search(r'
\s*
(.*)
\s*
', md.toc, re.S).group(1)
   post.toc = md.toc if m is not '' else None   # 为post增添一个属性，将目录赋给它，然后在detail中渲染它,toc是html代码
   return render(request, 'blog/detail.html', context={'post':post})

def archive(request,year,month):
   post_list = Post.objects.filter(created_time__year=year,created_time__month=month   # django的参数列表要求将.改为__
                                   ).order_by('-created_time')
   return render(request,'blog/index.html',context={'post_list': post_list})

def category(request,pk):
   cate = get_object_or_404(Category,pk = pk)
   post_list = Post.objects.filter(category=cate).order_by('-created_time')
   return render(request,'blog/index.html',context={'post_list':post_list})


from rest_framework.permissions import BasePermission
from rest_framework.request import Request

class TempPermissions(BasePermission):
    """TempPermissions."""

    def __init__(self) -> None:
        self._admin: bool = False
        self._user: bool = False

    def _init_permissions(self, request: Request) -> None:

        self._user = (
            request.user and
            request.user.is_active
        )
        self._admin = self._user and (
            request.user.is_staff and
            request.user.is_superuser
        )

    def has_permission(self, request: Request, view: 'TempViewSet') -> bool:

        self._init_permissions(request)

        basic_endpoints: list[str] = [
            'list',
            'list-2',
            'retrieve'
        ]
        advanced_endpoints: list[str] = [
            'create',
            'partial_update',
            'update',
            'destroy'
        ]
        if view.action in basic_endpoints:
            return self._user

        if view.action in advanced_endpoints:
            return self._admin

        return False