import cv2
import numpy as np
#img = cv2.imread("shapes/rectangle.png")
#img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
# cv.imshow("before transform",img)
#width, height = img.shape[:2]
#theta = -np.pi/4

# image_corners = np.array([
#     [0,0],[width,0],[width,height],[0,height]
# ], dtype=np.float32)

# transform_matrix = cv.getPerspectiveTransform(
#     src=image_corners,
#     dst=(image_corners+np.random.random(size=(4,2))*50).astype(np.float32)
# )

# print(transform_matrix)

def get_rotated_image(img: cv2.Mat, theta: float):
    '''
    Theta is the rotation direction, clockwise, in radians
    '''
    height, width = img.shape[:2]
    translate_to_center_matrix = np.array([
        [1,0,-width/2],
        [0,1,-height/2],
        [0,0,1]
    ])

    rotation_matrix = np.array([
            [np.cos(theta),-np.sin(theta),0],
            [np.sin(theta),np.cos(theta),0],
            [0,0,1]
        ],
        dtype=np.float32
    )

    new_w = np.ceil(width*np.abs(np.cos(theta))+height*np.abs(np.sin(theta))).astype(int)
    new_h = np.ceil(height*np.abs(np.cos(theta))+width*np.abs(np.sin(theta))).astype(int)

    undo_translate_to_center_matrix = np.array([
        [1,0,new_w/2],
        [0,1,new_h/2],
        [0,0,1]
    ])

    return cv2.warpPerspective(
    src=img,
    M=undo_translate_to_center_matrix@rotation_matrix@translate_to_center_matrix,
    dsize=(new_w,new_h)
)

def get_shape_bbox(img: cv2.Mat, show_box=False):
    '''
    img: cv2 Mat with shape (width,height, channels)

    Gets the bounding box for a shape on a completely black background (all background values are 0 exactly)
    Runtime is proportional to the number of pixels in the image (width*height).
    Returns a tuple (x_min, y_min, x_max, y_max)

    If show_box is True, this will use cv.imshow to display the bounding box on the image. It will use cv.waitKey(0) to wait for a keypress before dismissing the image display window.

    '''
    width,height = img.shape[:2]

    x_min, y_min = width, height
    x_max, y_max = 0,0

    for row in range(height):
        found_shape=False
        for col in range(width):
            if any(img[col][row]>0):
                x_min = min(col, x_min)
                x_max = max(col, x_max)
                found_shape=True
                break
        if found_shape:
            for col in reversed(range(width)):
                if any(img[col][row]>0):
                    x_min = min(col, x_min)
                    x_max = max(col, x_max)
                    break
            y_max = max(row, y_max)
            y_min = min(row, y_min)

    if show_box:
        with_rectangle = img.copy()
        cv2.rectangle(with_rectangle, (y_min, x_min),(y_max, x_max), (255,0,0))

        cv2.imshow("debug view of rotation",with_rectangle)
        cv2.waitKey(0)

    return (x_min, y_min, x_max, y_max)

if __name__=="__main__":
    warped = get_rotated_image(img,theta)
    box = get_shape_bbox(warped)
    print(box)


import numpy as np
import math
import sys
sys.path.append("../util/")
import get_mnist
import quickselect as q
from scipy.spatial.distance import euclidean

class kNN(object):
    set_size = 40
    def __init__(self, k, X_train, Y_train):
        self.k = k
        self.X_train = X_train
        self.Y_train = Y_train

        classes = {}

        for clazz in set(Y_train):
            assigned_samples = X_train[[x for x, label in enumerate(Y_train) if label == clazz]]
            classes[clazz] = np.mean(assigned_samples, axis=0)

        self.centroids = classes

    def predict(self, X_test):
        predictions = []
        knc = 0
        knn = 0
        for sample in X_test:
            distances = {clazz: euclidean(sample, centroid) for clazz, centroid in self.centroids.items()}
            vals = list(distances.values())
            v = np.array(vals).argsort()
            if distances[v[1]] - distances[v[0]] < distances[v[0]]/30:
                predictions.append(self.pred_knn(sample))
                knn += 1
            else:
                predictions.append(min(distances, key=distances.get))
                knc += 1

        print ("\n Centroids: " + str(knc))
        print ("Neighbors: " + str(knn))
        return predictions

    def pred_knn(self, sample):
        distances = [euclidean(x, sample) for x in self.X_train]
        tmp_dst = distances[:]
        k_smallest = q.select(tmp_dst, 0, len(tmp_dst) -1, self.k)
        labels = [self.Y_train[i] for i, dist in enumerate(distances)
                                                        if dist < k_smallest]
        return max(set(labels), key=labels.count)

# returns accuracy
def eval(preds, labels):
    correct = 0
    for i, prediction in enumerate(preds):
        if prediction == labels[i]:
            correct += 1

    return correct/len(labels)

if __name__ == "__main__":
    if len(sys.argv) > 1:
        k = int(sys.argv[1])
    else:
        k = 10

    num_dim = 200
    X_train, Y_train, X_test, Y_test = get_mnist.load_mnist()

    X_train = X_train[0:3000]
    Y_train = Y_train[0:3000]
    X_test = X_test[0:500]
    Y_test = Y_test[0:500]

    dim = len(X_train[0])

    print ("Reducing Dimensionality...")
    print (str(dim) + " -> " + str(num_dim))
    variances = np.var(X_train, axis=0)

    indices = variances.argsort()[:len(X_train[0])-num_dim]

    X_train = np.delete(X_train, indices, axis=1)
    X_test = np.delete(X_test, indices, axis=1)

    print ("Predicting test instances...", end="", flush=True)
    classifier = kNN(k, X_train, Y_train)
    predictions = classifier.predict(X_test)
    print ("Done")
    err_rate = 1 - eval(predictions, Y_test)

    print ("-----------")
    print ("Error Rate: " + str(err_rate*100) + "%")


class Document:
   def __init__(self):
       self.titulo = ""
       self.autor = ""
       self.fecha_de_actualizacion = ""
       self.contenido =  ""

   def __str__(self):                # comportamiento para iprint
       tmp = """
       Documento: %s
       Creado por: %s

       %s

       """ % (self. titulo, self.autor , self.contenido)
       return tmp

if __name__ == '__main__':
   A = Document()
   A.titulo = "Mi primer objeto"
   A.autor = "Chris"
   A.contenido = "Este es un ejemplo de clase simple"
   print(A)


from selenium.webdriver.common.by import By

from Pages.BasePage import BasePage

class Page(BasePage):
    element_locator1 = (By.ID, 'locator')
    element_locator2 = (By.XPATH, 'locator')
    element_locator3 = (By.CLASS_NAME, 'locator')
    element_locator4 = (By.CSS_SELECTOR, 'locator')
    element_locator5 = (By.NAME, 'locator')

    def __init__(self, driver, wait, action):
        super().__init__(driver, wait, action)


##############################################################################################
#                                # Server Manager GUI #                                      #
##############################################################################################
# TODO's for myself ->>
#
# TODO>> Add popups on every network button
# TODO>> terminal ssh
# TODO>> do the popup gui
#
##############################################################################################

import customtkinter as ctk
import os
import socket
import speedtest
import pywifi
import json
import platform
import subprocess
import threading
import netifaces
try: # The error handler
    class App(ctk.CTk):
        def __init__(self):
            # INIT of Self
            super().__init__()

            # Window config
            self.title("servmanGUI")
            self.geometry(f"{600}x{400}")
            self.resizable(False,False)

            # Variables
            main_tab_Home_about_text_var = """This tool uses ssh and internet connection to find every single
    open port or responding IP that you want to find.
    Also it can find any router ip if you forgot.

            """

            self.ip = self.get_ip()
            self.subnet = ".".join(self.ip.split('.')[:-1])

            config_raw_r=json.load(open("config.json","r"))
            self.num_threads = config_raw_r["num_threads"]
            self.ip_start = config_raw_r["start_ip"]
            self.ip_end = config_raw_r["end_ip"]
            self.ip_buttons_ids = []
            self.toplevel_window = None

            # Frames
            self.main_left = ctk.CTkFrame(self,width=200,height=400)
            self.main_tab = ctk.CTkTabview(self,width=400,height=400,fg_color="black")

            # Fonts
            self.main_left_curr_wifi_conn_font = ctk.CTkFont(family="Connection_font",size=20,weight="bold")
            self.Big_text = ctk.CTkFont(family="big_text",size=30,weight="bold")

            # Tabs
            self.main_tab.add("Home")
            self.main_tab.add("Network")
            self.main_tab.add("Remote Terminal")

            # Left Frame Info
            self.main_left_curr_wifi_conn = ctk.CTkLabel(self.main_left, text="Wait",width=200,text_color="white", font=self.main_left_curr_wifi_conn_font)
            self.main_left_curr_wifi_speed = ctk.CTkLabel(self.main_left, text="Speed: Wait")
            self.main_left_curr_wifi_LAN_ip = ctk.CTkLabel(self.main_left, text="IP: Wait")
            # Right Tabs

            ## Home Tab
            self.main_tab_Home_welcome_label = ctk.CTkLabel(self.main_tab.tab("Home"),text=f"Welcome, {os.getlogin()}",font=self.Big_text,width=387)
            self.main_tab_Home_about_text = ctk.CTkLabel(self.main_tab.tab("Home"),text=main_tab_Home_about_text_var,width=387)

            ## Network Tab
            self.main_tab_Network_refresh_button = ctk.CTkButton(self.main_tab.tab("Network"),text="refresh",width=387,command=self.refresh_button_middleman)
            # Grid Applying
            self.main_left.grid(row=0,column=0,sticky="n"+"w"+"s"+"e")
            self.main_left_curr_wifi_conn.grid(row=0,column=0,sticky="w"+"e"+"n")
            self.main_left_curr_wifi_speed.grid(row=1,column=0,sticky="w"+"e"+"n")
            self.main_left_curr_wifi_LAN_ip.grid(row=2,column=0)
            self.main_tab.grid(row=0,column=1,sticky="n"+"w"+"s"+"e")
            self.main_tab_Home_welcome_label.grid(row=0,column=0,sticky="n")
            self.main_tab_Home_about_text.grid(row=1,column=0,sticky="n")
            self.main_tab_Network_refresh_button.grid(row=0,column=0)

        # Logic

        ## Functions
        def get_ssid(self):
            wifi = pywifi.PyWiFi()
            iface = wifi.interfaces()[0]  # Assuming you have one WiFi interface
            ssid = iface.scan_results()[0].ssid
            return ssid

        def get_ip(self):
            try:
                # Create a socket to the Google DNS server (8.8.8.8)
                s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                s.connect(("8.8.8.8", 80))
                local_ip = s.getsockname()[0]
                s.close()
                return local_ip

            except socket.error:
                return None

        def truncate_float(self,float_number, decimal_places):
            multiplier = 10 ** decimal_places
            return int(float_number * multiplier) / multiplier

        def get_Up_Down(self):
            try:
                st = speedtest.Speedtest()
                download_speed = st.download()/1000000  # Convert to Mbps
                upload_speed = st.upload()/1000000  # Convert to Mbps
                return  "DOWN:"+str(self.truncate_float(download_speed,2)) +" // "+"UP:"+str(self.truncate_float(upload_speed,2))

            except Exception as e:
                print(f"EXCEPTION OCCURED: {e} , Contact me ASAP!")
                return "Disconnected"

        def check_ip_existence(self,ip, result_list):
            try:
                # Use the 'ping' command on Linux or Windows to check if the IP exists
                if platform.system() == "Linux":
                    print(f"checking {ip}")
                    output = subprocess.check_output(["ping", "-c", "1", ip], stderr=subprocess.STDOUT, text=True)
                    if "1 packets transmitted," in output and "0 received" not in output:
                        result_list.append(ip)
                        print(f"Checked IP: {ip}")
                elif platform.system() == "Windows":
                    output = subprocess.check_output(["ping", "-n", "1", ip], stderr=subprocess.STDOUT, text=True)
                    if "Received = 1" in output:
                        result_list.append(ip)
                        print(f"Checked IP: {ip}")
            except subprocess.CalledProcessError as e:
                pass
            except Exception as e:
                print(f"Error checking IP {ip}: {e}")

        def scan_lan_ips(self,subnet, num_threads=4,start_ip = 1,end_ip = 255):
            # Create a list to store results
            if end_ip - start_ip + 1 < num_threads:
                raise ValueError("Must be same or less threads than checked ip adresses")
            result_list = []

            # Calculate the number of IPs each thread should check
            ips_per_thread = (end_ip - start_ip + 1) // num_threads

            # Create and start threads
            try:
                threads = []
                for i in range(num_threads):
                    start = start_ip + i * ips_per_thread
                    end = start + ips_per_thread - 1
                    thread = threading.Thread(target=self.check_ip_range, args=(subnet, start, end, result_list))
                    thread.start()
                    threads.append(thread)

                # Wait for all threads to finish
                for thread in threads:
                    thread.join()

                return result_list
            except KeyboardInterrupt:
                print("DO NOT INTERRUPT THE THREADS")
            except Exception as e:
                print("Whoops, we got an exception: "+e)
        def check_ip_range(self,subnet, start, end, result_list):
            for i in range(start, end + 1):
                ip = subnet + "." + str(i)
                self.check_ip_existence(ip, result_list)

        def update_network_list(self,window,button_labels):
            for button in self.ip_buttons_ids:
                button.destroy()
            self.ip_buttons_ids.clear()
            for i, label in enumerate(button_labels):
                button = ctk.CTkButton(window, text=label,anchor="w",width=387,fg_color="gray",command=self.open_toplevel)
                button.grid(row=i+1, column=0,sticky="w")
                self.ip_buttons_ids.append(button)

        def get_router_gateway_ip():
            try:
                # Get the default gateway's IP address (cross-platform)
                gateways = netifaces.gateways()
                if 'default' in gateways and netifaces.AF_INET in gateways['default']:
                    return gateways['default'][netifaces.AF_INET][0]
                return None
            except Exception as e:
                print(f"Error getting router gateway IP: {e}")
                return None
        def refresh_button_middleman(self):
            try:
                self.update_network_list(self.main_tab.tab("Network"),self.scan_lan_ips(subnet=self.subnet,num_threads=self.num_threads,start_ip=self.ip_start,end_ip=self.ip_end))
            except Exception as e:
                print(f"whoops? {e}")

        ## Updating Functions
        def main_left_update_ip(self):
            self.main_left_curr_wifi_LAN_ip.configure(text=f"IP: {self.ip}")
            app.after(1000,self.main_left_update_ip)

        def main_left_update_ssid(self):
            self.main_left_curr_wifi_conn.configure(text=f"{self.get_ssid()}")
            app.after(1000,self.main_left_update_ssid)

        def main_left_update_speed(self):
            self.main_left_curr_wifi_speed.configure(text=self.get_Up_Down())
        def open_toplevel(self):
                if self.toplevel_window is None or not self.toplevel_window.winfo_exists():
                    self.toplevel_window = ToplevelWindow(self)  # create window if its None or destroyed
                else:
                    self.toplevel_window.focus()  # if window exists focus it

    # IP window
    class ToplevelWindow(ctk.CTkToplevel):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.geometry("400x300")

            self.label = ctk.CTkLabel(self, text="ToplevelWindow")
            self.label.pack(padx=20, pady=20)

    if __name__ == "__main__":
        app = App()
        app.main_left_update_ip()
        app.main_left_update_ssid()
        speedthread = threading.Thread(target=app.main_left_update_speed)
        speedthread.setDaemon(True)
        speedthread.start()
        app.mainloop()

except PermissionError:
    print("Rerun with sudo/Administrator privs")

except KeyboardInterrupt:
    print("See you next time :3")
except Exception as e:
    print(f"Whoops, you got an main thread error: {e}")


try:
    from ConfigParser import SafeConfigParser
except ImportError:
    from configparser import SafeConfigParser
import os

from .constants import LICENSES
from .loaders import URLLoader, FileSystemLoader, ChannelLoader
from .utils import clean_dict, recursive_update

DEFAULTS = {
    'config_file': os.path.expanduser('~/.pypackager/pypackager.cfg'),
    'template': {
        'dir': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'template'),
        'syntax': 'pystache'
    },
    'index': 'https://raw.github.com/fcurella/pypackager-channel/master/index.json',
    'template-loaders': (
        URLLoader,
        FileSystemLoader,
        ChannelLoader
    ),
    'templates_dir': os.path.join(os.path.expanduser('~/.pypackager'), 'templates'),
    'template_wrap_dir': 'template',
    'support_dir': os.path.expanduser('~/.pypackager'),
}

class RecursiveDict(dict):
    def recursive_update(self, kwargs):
        self = recursive_update(self, clean_dict(kwargs))
        return self

class SettingsLoader(RecursiveDict):
    def parse(self, config_file):
        kwargs = {}
        parser = SafeConfigParser()

        if os.path.exists(config_file):
            with open(config_file) as fh:
                parser.readfp(fh)

            for section in parser.sections():
                kwargs[section] = recursive_update(kwargs.get(section, {}), dict(parser.items(section)))

        if 'license' in kwargs and 'type' in kwargs['license']:
            license_shortname = kwargs['license']['type']
            if license_shortname in LICENSES:
                kwargs['license']['classifier'] = LICENSES[license_shortname]
                kwargs['license']['name'] = LICENSES[license_shortname].split('::')[-1].strip()
            else:
                kwargs['license']['classifier'] = LICENSES['other']
                kwargs['license']['name'] = license_shortname

        return kwargs

class DefaultsLoader(SettingsLoader):
    def __init__(self):
        super(DefaultsLoader, self).__init__(**DEFAULTS)

class UserSettingLoader(SettingsLoader):
    config_file = DEFAULTS['config_file']

    def __init__(self, config_file=None):
        if config_file is not None:
            self.config_file = config_file

        kwargs = self.parse(self.config_file)

        super(UserSettingLoader, self).__init__(**kwargs)

class TemplateSettingLoader(SettingsLoader):
    templates_dir = DEFAULTS['templates_dir']

    def __init__(self, template, loaders, settings):
        kwargs = {}

        if 'templates_dir' not in settings:
            settings['templates_dir'] = self.templates_dir

        template_dir = None
        for Loader in loaders:
            loader = Loader(settings, template)
            if loader.template_exists():
                template_dir = loader.template_path()
                break

        package_cfg = os.path.join(template_dir, '.package.cfg')
        kwargs = self.parse(package_cfg)
        super(TemplateSettingLoader, self).__init__(**kwargs)

class CommandLineSettingLoader(SettingsLoader):
    pass

class SettingsReader(dict):
    support_dir = os.path.expanduser('~/.pypackager')

    def __init__(self, config_file=None, *args, **kwargs):
        _kwargs = RecursiveDict()

        defaults = DefaultsLoader()
        command_line = CommandLineSettingLoader(**kwargs)
        user = UserSettingLoader(config_file)

        if 'index' in command_line:
            index = command_line['index']
        else:
            index = defaults['index']

        template_loader_settings = {
            'index': index,
            'template_wrap_dir': DEFAULTS['template_wrap_dir'],
            'support_dir': DEFAULTS['support_dir']
        }

        if 'template' in command_line and 'dir' in command_line['template']:
            template_dir = command_line['template']['dir']
        else:
            template_dir = defaults['template']['dir']

        template = TemplateSettingLoader(template_dir, defaults['template-loaders'], template_loader_settings)

        _kwargs.recursive_update(defaults)
        _kwargs.recursive_update(template)
        _kwargs.recursive_update(user)
        _kwargs.recursive_update(command_line)

        super(SettingsReader, self).__init__(**_kwargs)


from flask import Flask, render_template

application = Flask(__name__)

@application.route('/')
def index():
    poto = [
        {
            'nama' : '1. Kim Junkyu',
            'image': 'images/img7.jpg'
        },
        {
            'nama' : '2. Haruto',
            'image': 'images/img2.jpg'
        },
        {
            'nama' : '3. Haruto 2',
            'image': 'images/img3.jpg'
        },
        {
            'nama' : '4. Treasure',
            'image': 'images/img4.jpg'
        },
        {
            'nama' : '5. Treasure',
            'image': 'images/img5.jpg'
        }
    ]

    return render_template('index.html',poto=poto)
if __name__ == '__main__':
    application.run(debug=True)


import obspy,sys,os,glob
import numpy as np
import pyfftw
from scipy.signal import spectrogram
from numpy import linalg as LA
from obspy import UTCDateTime
import matplotlib.pyplot as plt

def trIntegrate(trIn):
    from scipy.fftpack import diff
    trOut = trIn.copy()
    trOut.data = diff(trOut.data,-1,trOut.stats.npts*trOut.stats.delta)
    return trOut

def trDifferentiate(trIn):
    from scipy.fftpack import diff
    trOut = trIn.copy()
    trOut.data = diff(trOut.data,1,trOut.stats.npts*trOut.stats.delta)
    return trOut

def doResample(st,dt,npts=None):
    st.filter('lowpass',corners=6,freq=0.5/dt/2,zerophase=True)
    starttime = st[0].stats.starttime
    endtime = st[0].stats.endtime
    for tr in st:
        if tr.stats.starttime > starttime:
            starttime = tr.stats.starttime
        if tr.stats.endtime < endtime:
            endtime = tr.stats.endtime
    if npts is None:
        npts = int(round((endtime - starttime)/dt)) - 5
    st.interpolate(1.0/dt,starttime=starttime,npts=npts)
    return st

def nextpow2(L):
    N = 1
    while N < L:
        N *= 2
    return N

def flatHann(N=256,taper=0.3):
    """ Flat Hanning window, taper at both sides

    Args:
        N:      length of flat hanning window
        taper:  normalized length to be tapered at each side

    Returns:
        flat hanning window
    """
    win = np.ones(N)
    ntaper = int(N*taper)
    i = np.arange(2*ntaper)
    winHann = 0.5-0.5*np.cos(2*np.pi*i/(len(i)-1))
    win[:ntaper] = winHann[:ntaper]
    win[-ntaper:] = winHann[-ntaper:]
    return win

def smooth(a,N=5):
    ''' smooth function, similar to smooth function smooth(y,span) in Matlab(R2018a) '''
    if N%2 != 1:
        N = N-1
    b = np.convolve(a, np.ones(N)/N, mode='same')
    for i in range((N-1)//2):
        b[i]        = a[:(2*i+1)].mean()
        b[-1-i]     = a[(-1-2*i):].mean()
    return b

def ftest(res1,parms1,res2,parms2):
    ''' Function to perform a formal F-test
        function to perform a formal F-test on two sets of residuals of fits to
        data (res1, res2) that have respectively been fitted using parms1,parms2
        (where these are integers equal to the number of parameters used in each
        fit)
        IMPORTANTLY:
        This will test whether the second fit (yielding res2) is statistically
        superior to the first fit, but used more parms. I.e.:
                sum(abs(res2)) < sum(abs(res1))
        and            parms2 > parms1

        the degrees of freedom for each fit are therefore:
        1) N - parms1
        2) N - parms2
        where N is the number of data, or length(res*)
        The residuals are just equal to dobs-dpred, so we square and sum these to
        get the chi^2 values (Ea)
        by Z. Eilon, python by Ayu Tsukimaya
    '''
    from scipy.stats import f
    res1 = np.array(res1)
    res2 = np.array(res2)
    N1, N2 = len(res1), len(res2)
    v1, v2 = N1-parms1, N2-parms2 # degrees of freedom
    Ea_1, Ea_2 = (res1*res1).sum(), (res2*res2).sum() # calculate chi^2 sums

    if Ea_1 == 0 or Ea_2 == 0 or v1 == 0 or v2 == 0:
        return 1
    else:
        Fobs = (Ea_1/v1)/(Ea_2/v2)
        return 1 - ( f.cdf(Fobs,v1,v2) - f.cdf(1/Fobs,v1,v2) )

def QC_staspectra_windows(trZ,tr1,tr2,trP,segT,overlap,passband,thres=1.5,saveFigs=False,
                          figDir=None):
    """ Function to identify time segments that potential transient signal exists
    Args:
        trZ,tr1,tr2,trP: traces of Z, two horizontal and P component
        segT:            segment(window) length, in seconds
        overlap:         normalized (1 = segT) overlap length of segments
        passband:        frequency band used in this function
        thres:           segment > STD*thres is suspect including transient signal
        saveFigs:        if you want to save new spectrogram and spectrum figures
        figDir:          directory to save figures
    Returns:
        winBs,winEs:    index of start and end point of each segment
        goodwins:       index of good segments
    """
    spectra = []
    spectraPlot = []
    for tr in [trZ,tr1,tr2,trP]:
        npts = round(segT/tr.stats.delta)
        f,t,sxx = spectrogram(tr.data, fs=1.0/tr.stats.delta,
                    window=flatHann(N=npts,taper=min(0.3,200/segT)),
                    # window: Helen use 0.3, Ye use min(0.4,200/segT)
                    nperseg=npts, noverlap=int(npts*0.3), nfft=npts,mode='complex')
        sxx[sxx==0] = np.finfo(float).eps
        sxx = np.log10(abs(sxx))
        if saveFigs:
            spectraPlot.append(sxx.copy())
        sxx[(f<passband[0]) + (f>passband[1]) > 0,:] = 0
        for i in range(sxx.shape[1]):
            sxx[:,i] = smooth(sxx[:,i],N=50)
        for i in range(sxx.shape[0]):
            sxx[i,:] = sxx[i,:] - sxx[i,:].mean()
        spectra.append(sxx)
    winBs = np.round((t - segT/2) / trZ.stats.delta).astype(int)
    winEs = winBs + npts-1
    winN  = len(t)

    goodwins = np.array(list(range(winN)))
    moveon = False
    while moveon == False:
        normStd = []
        for win in goodwins:
            tmpIndex = [True if i in goodwins else False for i in range(winN) ]
            tmpIndex[win] = False
            normStd.append( [LA.norm(spectra[i][:,tmpIndex].std(axis=1)) for i in range(len(spectra))] )
        penalty = -(normStd - np.median(normStd,axis=0)).sum(axis=1)
        kill = penalty>thres*penalty.std()
        if np.any(kill) == False:
            break
        trypenalty = penalty[~kill]

        if ftest(penalty,1,trypenalty,1) < 0.05: # test with 95% confidence interva
            goodwins = goodwins[~kill]
            moveon = False
        else:
            moveon = True # no reason to remove more windows

    if saveFigs and (figDir is not None):
        fig,axes = plt.subplots(5,1,sharex=True,figsize=[8,10.5])
        nmDict = ['Z','H1','H2','P']
        for i in range(4):
            axes[i].pcolormesh(t,f,spectraPlot[i],cmap='jet')
            axes[i].set_title(nmDict[i])
            axes[i].set_ylabel('Frequency (Hz)')
        goodwinsInd = [i in goodwins for i in range(winN)]
        axes[-1].plot(t[goodwinsInd],np.ones(len(goodwins)),'kp')
        outfig = f'{trZ.stats.network}.{trZ.stats.station}.{trZ.stats.starttime.strftime("%Y%m%d%H%M%S")}'
        os.makedirs(f'{figDir}/STATIONS_DAILY',exist_ok=True)
        plt.savefig(f'{figDir}/STATIONS_DAILY/{outfig}.spectrogram.png')
        plt.close(fig.number)

    return [winBs,winEs,goodwins]

def calSpectra(trZ,tr1,tr2,trP,dsetDir,segT=3600.0, overlap=0.3, minwin=10, minLen=75000, trType='disp',
        QCpassband=None, saveSpec=True, calrotation=False, saveFigs=False, verbose=True, prefilt=None):
    """ Calculate spectrums after transient signal segments are removed
    Args:
        trZ,tr1,tr2,trP: traces of Z, two horizontal and P component
        dsetDir:         directory to save all results got in this module
        segT:            segment(window) length, in seconds
        overlap:         normalized (1 = segT) overlap length of segments
        minwin:          minimum number of segments required to continue decoupling.
        calrotation:     calculate orientation or not
        saveSpec:        save spectrums or not
        saveFigs:        save figures or not
    Returns:
        specprop:        spectrums and related propties
    """
    if QCpassband is None:
        passband = [0.01, .2]  # frequency band to check transient signal
                                # Ye Tian's setting: [1.0/20,1.0/11]
                                # Helen's setting:[0.004, .2]
    else:
        passband = QCpassband
    tiltfreq = [.005, .035]     # frequency band to check orientation
    threshold= 1.5              # a threshold to check transient signal
    figDir = f'{dsetDir}/FIGURES'

    if type(trZ)==str:
        trZ,tr1,tr2,trP = [obspy.read(trZ)[0],obspy.read(tr1)[0],
                           obspy.read(tr2)[0],obspy.read(trP)[0]]
    if prefilt is not None:
        for tr in [trZ,tr1,tr2,trP]:
            tr.detrend('linear')
            tr.taper(0.1,max_length=500)
            tr.filter('bandpass',freqmin=prefilt[0],freqmax=prefilt[1],zerophase=True)

    if trType == 'disp':
        pass
    elif trType == 'vel':
        # trZ.integrate()           # obspy.integrate is different from integral with FFT
        # tr1.integrate()           # but similar when you apply detrend+taper+highpass filter.
        # tr2.integrate()
        trZ = trIntegrate(trZ)
        tr1 = trIntegrate(tr1)
        tr2 = trIntegrate(tr2)
    else:
        raise ValueError(f'wrong trType:{trType}')

    # check traces
    for tr in [trZ,tr1,tr2,trP]:
        if tr.stats.sac.stel != trZ.stats.sac.stel:     # check elevation
            print('Different elevation found!')
            return False
        if tr.stats.starttime != trZ.stats.starttime:   # check starttime
            print('Different starttime found!')
            return False
        if tr.stats.delta != trZ.stats.delta:           # check sample rate
            print('Different sample rate found!')
            return False
        if tr.stats.npts != trZ.stats.npts:             # check npts
            print('Different npts found!')
            return False
        if tr.stats.npts*tr.stats.delta < minLen:      # check record length
            print('Record length is not long enough!')
            return False
        if np.all(tr.data == 0):                        # check if all-zero record
            print('All-zero record!')
            return False

    # select noise window
    [iptWinBs,iptWinEs,goodwins] = QC_staspectra_windows(trZ,tr1,tr2,trP,segT=segT,overlap=overlap,
                                            passband=passband,thres=threshold,saveFigs=saveFigs,
                                            figDir=figDir)
    if len(goodwins) < minwin:
        print(f'Not enough good data segments! {minwin} required but {len(goodwins)} got')
        return False
    else:
        if verbose:
            print(f'{len(goodwins)} good segments. Proceeding...')

    # calculate spectrum of all window
    Nwin = len(iptWinBs)
    goodwinsInd = np.array([i in goodwins for i in range(Nwin)])
    npts = iptWinEs[0]-iptWinBs[0]+1
    NFFT = nextpow2(npts)
    dt = trZ.stats.delta
    fs = 1.0/dt
    f =  fs/2*np.linspace(0,1,NFFT//2+1)
    specAll = [[],[],[],[]]
    for i in range(len(iptWinBs)):
        for j,tr in enumerate([trZ,tr1,tr2,trP]):
            data = tr.data[iptWinBs[i]:iptWinEs[i]+1]
            data = data*flatHann(N=len(data), taper=min(0.3,200/segT)) # Helen use 0.3, Ye use min(0.4,200/segT)
            data = data-data.mean()
            spectrum = np.fft.fft(data,NFFT) * tr.stats.delta
            spectrum = spectrum[:NFFT//2+1]
            specAll[j].append(spectrum)
    specZ,spec1,spec2,specP = specAll

    # calculate cross-correlations(CC) in frequency domain
    # get avg CC of noise window
    cZZ,c11,c22,cPP = [],[],[],[]
    c1Z,c2Z,cPZ,c1P,c2P,c12 = [],[],[],[],[],[]
    for i in range(len(iptWinBs)):
        cZZ.append((specZ[i]*specZ[i].conj()).real*2/(NFFT*dt))
        c11.append((spec1[i]*spec1[i].conj()).real*2/(NFFT*dt))
        c22.append((spec2[i]*spec2[i].conj()).real*2/(NFFT*dt))
        cPP.append((specP[i]*specP[i].conj()).real*2/(NFFT*dt))
        c1Z.append(spec1[i]*specZ[i].conj()*2/(NFFT*dt))
        c2Z.append(spec2[i]*specZ[i].conj()*2/(NFFT*dt))
        cPZ.append(specP[i]*specZ[i].conj()*2/(NFFT*dt))
        c1P.append(spec1[i]*specP[i].conj()*2/(NFFT*dt))
        c2P.append(spec2[i]*specP[i].conj()*2/(NFFT*dt))
        c12.append(spec1[i]*spec2[i].conj()*2/(NFFT*dt))
    power = {'cZZ':np.array(cZZ)[goodwinsInd].mean(axis=0),
            'c11':np.array(c11)[goodwinsInd].mean(axis=0),
            'c22':np.array(c22)[goodwinsInd].mean(axis=0),
            'cPP':np.array(cPP)[goodwinsInd].mean(axis=0)}
    cross = {'c1Z':np.array(c1Z)[goodwinsInd].mean(axis=0),
            'c2Z':np.array(c2Z)[goodwinsInd].mean(axis=0),
            'cPZ':np.array(cPZ)[goodwinsInd].mean(axis=0),
            'c1P':np.array(c1P)[goodwinsInd].mean(axis=0),
            'c2P':np.array(c2P)[goodwinsInd].mean(axis=0),
            'c12':np.array(c12)[goodwinsInd].mean(axis=0)}

    # set output
    specprop = {'power':power,'cross':cross}
    specprop['params'] = {'f':f, 'network':trZ.stats.network, 'station':trZ.stats.station,
                        'elev': trZ.stats.sac.stel,
                        'goodwins':goodwins,'iptWinBs':iptWinBs,'iptWinEs':iptWinEs,
                        'delta':dt,'overlap':overlap,'NFFT':NFFT}
    if saveFigs:
        def plotSpecWin(ax,f,spec,title):
            maxf = max(f)
            N = len(spec)
            f = np.tile(f,(N,1))
            spec = np.array([smooth(spec[i],N=100) for i in range(N)])
            ax.loglog(f.T,spec.T,'r',lw=0.5)
            ax.loglog(f[goodwinsInd].T,spec[goodwinsInd].T,'k',lw=0.5)
            ax.set_xlim(1/250,maxf)
            ax.set_title(title)
            ax.set_ylabel('PSD (m^2/Hz)')
        def plotSpecAvg(ax,f,spec,title):
            maxf = max(f)
            spec = smooth(spec,N=100)
            ax.loglog(f,spec,'k',lw=1.0)
            ax.set_xlim(1/250,maxf)
            ax.set_title(title)
        fig,axes = plt.subplots(4,2,gridspec_kw={'hspace':0.4},figsize=[8,10.5])
        plotSpecWin(axes[0,0],f,cZZ,'Z - all windows')
        plotSpecWin(axes[1,0],f,c11,'H1 - all windows')
        plotSpecWin(axes[2,0],f,c22,'H2 - all windows')
        plotSpecWin(axes[3,0],f,cPP,'P - all windows')
        plotSpecAvg(axes[0,1],f,power['cZZ'],'Z - Daily Average')
        plotSpecAvg(axes[1,1],f,power['c11'],'H1 - Daily Average')
        plotSpecAvg(axes[2,1],f,power['c22'],'H2 - Daily Average')
        plotSpecAvg(axes[3,1],f,power['cPP'],'P - Daily Average')
        outfname = f'{trZ.stats.network}.{trZ.stats.station}.{trZ.stats.starttime.strftime("%Y%m%d%H%M%S")}'
        os.makedirs(f'{dsetDir}/FIGURES/STATIONS_DAILY',exist_ok=True)
        plt.savefig(f'{dsetDir}/FIGURES/STATIONS_DAILY/{outfname}.dailySpectrum.png')
        plt.close()

    if calrotation:
        def estimateRotation(angles):
            angles = angles/180.0*np.pi
            cohLst,phLst = [],[]
            for ang in angles:
                specH = [np.sin(ang)*spec2[i]+np.cos(ang)*spec1[i] for i in range(len(iptWinBs))]
                cHH,cHZ = [],[]
                for i in range(len(iptWinBs)):
                    if i not in goodwins:
                        continue
                    cHH.append(abs(specH[i]*specH[i]).conj()*2/(NFFT*dt))
                    cHZ.append(specH[i]*specZ[i].conj()*2/(NFFT*dt))
                cHHAvg = np.array(cHH).mean(axis=0)
                cHZAvg = np.array(cHZ).mean(axis=0)
                cZZAvg = power['cZZ']

                cohAvg = (abs(cHZAvg)*abs(cHZAvg))/(cHHAvg*cZZAvg)
                phAvg = 180/np.pi*np.arctan2(cHZAvg.imag,cHZAvg.real)
                adAvg = abs(cHZAvg)/cHHAvg

                ind = (f>=tiltfreq[0]) * (f<=tiltfreq[1])
                cohLst.append(cohAvg[ind].mean())
                phLst.append(abs(phAvg[ind]).mean())
            ind = np.abs(phLst) < 90
            ind = np.where(ind)[0][np.argmax(np.array(cohLst)[ind])]
            max_coh = cohLst[ind]
            max_ori = angles[ind]/np.pi*180
            return (max_coh,max_ori,phLst,cohLst)

        angles = np.arange(0,360,10)
        max_coh,max_ori,phLst,cohLst = estimateRotation(angles)
        angles = np.arange(max_ori-10,max_ori+11,1)
        max_coh,max_ori,_,_ = estimateRotation(angles)
        specprop['params']['rotor'] = max_ori
        specprop['params']['rotcoh'] = max_coh

        ang = max_ori
        specH = [np.sin(ang)*spec2[i]+np.cos(ang)*spec1[i] for i in range(len(iptWinBs))]
        cHH,cHZ,cHP = [],[],[]
        for i in range(len(iptWinBs)):
            if i not in goodwins:
                continue
            cHH.append(specH[i]*specH[i].conj()*2/(NFFT*dt))
            cHZ.append(specH[i]*specZ[i].conj()*2/(NFFT*dt))
            cHP.append(specH[i]*specP[i].conj()*2/(NFFT*dt))
        cHHAvg = np.array(cHH).mean(axis=0)
        cHZAvg = np.array(cHZ).mean(axis=0)
        cHPAvg = np.array(cHP).mean(axis=0)
        specprop['rotation'] = {'cHH':cHHAvg,'cHZ':cHZAvg,'cHP':cHPAvg}

    if saveSpec:
        staID = f'{trZ.stats.network}.{trZ.stats.station}'
        os.makedirs(f'{dsetDir}/SPECTRA/{staID}',exist_ok=True)
        outfname = f'{staID}.{trZ.stats.starttime.strftime("%Y%m%d%H%M%S")}.spectra.npz'
        np.savez_compressed(f'{dsetDir}/SPECTRA/{staID}/{outfname}',specprop=specprop)

    return specprop

def QC_cleanstaspectra_days(cZZs,c11s,c22s,cPPs,f,passband,thres=1.5):
    """ Second quality control for ensemble of spectrums:
    Args:
        cZZs,c11s,c22s,cPPs: list of self-correlation in frequency domain of Z,
                             two horizontal and P component
        f: frequencies
        passband: frequency band used to do quality control
        thres: spectrums > STD*thres is suspect as outliers
    Returns:
        isGood: list of bools to indicate which spectrums are good
    """
    N = len(cZZs)
    spec = np.array([cZZs,c11s,c22s,cPPs])
    spec[spec==0] = np.finfo(float).eps
    spec = np.log10(abs(spec))
    spec[ :,:,(f>passband[1]) + (f<passband[0]) > 0 ] = 0
    for i in range(spec.shape[0]):
        for j in range(spec.shape[1]):
            spec[i,j,:] = smooth(spec[i,j,:],50)
        spec[i,:,:] = spec[i,:,:] - spec[i,:,:].mean(axis=0)

    goodLst = np.arange(N)
    moveon = False
    while moveon == False:
        normStd = []
        for igood in goodLst:
            tmpisgood = [True if i in goodLst else False for i in range(N)]
            tmpisgood[igood] = False
            normStd.append( [LA.norm(spec[i][tmpisgood,:].std(axis=1)) for i in range(len(spec))] )
        normStd = np.array(normStd)
        penalty = -(normStd - np.median(normStd,axis=0)).sum(axis=1)
        kill = penalty>thres*penalty.std()
        if np.any(kill) == False:
            break
        trypenalty = penalty[~kill]

        if ftest(penalty,1,trypenalty,1) < 0.05: # test with 95% confidence interva
            goodLst = goodLst[~kill]
            moveon = False
        else:
            moveon = True # no reason to remove more windows

    isGood = np.array([True if i in goodLst else False for i in range(N)])
    return isGood

def cleanSpectra(dsetDir,staID,saveAvg=True,saveFigs=True,QCpassband=None):
    """ check which spectrums/days are outliers, and calculate a average spectrum
    Args:
        dsetDir: dataset directory to save results got in this module
        staID: {network name}.{station name}
        saveAvg: save average spectrum or not
        svaeFigs: savefigs or not
    Returns:
        specprop: average specturm and related properties
    """
    if QCpassband is None:
        passband = [0.01, .2]  # frequency band to check transient signal
                                # Ye Tian's setting: [1.0/20,1.0/11]
                                # Helen's setting:[0.004, .2]
    else:
        passband = QCpassband
    thres= 1.5 # a threshold to check transient signal

    specprops = []
    specfiles = glob.glob(f'{dsetDir}/SPECTRA/{staID}/*.spectra.npz')
    for specfile in specfiles:
        specprops.append(np.load(specfile,allow_pickle=True)['specprop'][()])
        if specprops[0]['params']['NFFT'] != specprops[-1]['params']['NFFT']:
            raise ValueError()

    network = specprops[0]['params']['network']
    station = specprops[0]['params']['station']
    elev    = specprops[0]['params']['elev']
    f       = specprops[0]['params']['f']
    N       = len(specprops)
    Nwins   = [len(spec['params']['goodwins']) for spec in specprops]
    cZZs    = np.array([spec['power']['cZZ'] for spec in specprops])
    c11s    = np.array([spec['power']['c11'] for spec in specprops])
    c22s    = np.array([spec['power']['c22'] for spec in specprops])
    cPPs    = np.array([spec['power']['cPP'] for spec in specprops])
    c1Zs    = np.array([spec['cross']['c1Z'] for spec in specprops])
    c2Zs    = np.array([spec['cross']['c2Z'] for spec in specprops])
    cPZs    = np.array([spec['cross']['cPZ'] for spec in specprops])
    c1Ps    = np.array([spec['cross']['c1P'] for spec in specprops])
    c2Ps    = np.array([spec['cross']['c2P'] for spec in specprops])
    c12s    = np.array([spec['cross']['c12'] for spec in specprops])

    isGood = QC_cleanstaspectra_days(cZZs,c11s,c22s,cPPs,f,passband=passband,thres=thres)

    avgLst = []
    stdLst = []
    for cs in [cZZs,c11s,c22s,cPPs,c1Zs,c2Zs,cPZs,c1Ps,c2Ps,c12s]:
        cs = np.array(cs)
        csAvg = np.average(cs,axis=0,weights=isGood*np.array(Nwins))
        csStd = np.average((cs-csAvg)*(cs-csAvg),axis=0,weights=isGood*np.array(Nwins))
        csStd = csStd/(isGood.sum()-1+0.001)*isGood.sum()
        # https://www.itl.nist.gov/div898/software/dataplot/refman2/ch2/weightsd.pdf
        avgLst.append(csAvg)
        stdLst.append(csStd)
    power = {'cZZ':avgLst[0],'c11':avgLst[1],'c22':avgLst[2],'cPP':avgLst[3],
            'cZZ-std':stdLst[0],'c11-std':stdLst[1],'c22-std':stdLst[2],'cPP-std':stdLst[3]}
    cross = {'c1Z':avgLst[4],'c2Z':avgLst[5],'cPZ':avgLst[6],
            'c1P':avgLst[7],'c2P':avgLst[8],'c12':avgLst[9],
            'c1Z-std':stdLst[4],'c2Z-std':stdLst[5],'cPZ-std':stdLst[6],
            'c1P-std':stdLst[7],'c2P-std':stdLst[8],'c12-std':stdLst[9]}
    params = {'f':f,'network':network,'station':station,'elev':elev}
    specprop = {'params':params,'power':power,'cross':cross}

    if saveFigs:
        os.makedirs(f'{dsetDir}/FIGURES/STATIONS_NOISEPROP',exist_ok=True)
        maxpow = [np.array(cs)[:,:-1].max()*100 for cs in [cZZs,c11s,c22s,cPPs]] # avoid Nyquist Frequency
        minpow = [np.array(cs)[:,:-1].min()/10  for cs in [cZZs,c11s,c22s,cPPs]]
        # plot power spectrum
        fig,axes = plt.subplots(4,1,sharex=True,figsize=[8,10.5])
        for cs,title,i in zip([cZZs,c11s,c22s,cPPs],['Z','H1','H2','P'],range(4)):
            cs = np.array(cs)
            for j in range(len(cs)):
                cs[j] = smooth(cs[j],40)
            axes[i].loglog(np.tile(f,(len(cs),1)).T,cs.T,c=[0.5,0.5,0.5],lw=0.5)
            cs = cs[~isGood]
            axes[i].loglog(np.tile(f,(len(cs),1)).T,cs.T,c=[1,0,0],lw=0.5)
            axes[i].set_ylim(minpow[i],maxpow[i])
            axes[i].set_xlim(1e-4,max(f))
            axes[i].set_ylabel('Power (db)')
            axes[i].set_title(title)
        axes[-1].set_xlabel('Frequency (Hz)')
        plt.savefig(f'{dsetDir}/FIGURES/STATIONS_NOISEPROP/{staID}.spectrum.png')
        plt.close()
        # plot Coherence,Phase,admittance
        coh = [abs(c1Zs)**2/(c11s*cZZs),
            abs(c2Zs)**2/(c22s*cZZs),
            abs(cPZs)**2/(cPPs*cZZs),
            abs(c1Ps)**2/(c11s*cPPs),
            abs(c2Ps)**2/(c22s*cPPs),
            abs(c12s)**2/(c11s*c22s)]
        phase = [np.arctan2(c1Zs.imag,c1Zs.real)/np.pi*180.0,
                np.arctan2(c2Zs.imag,c2Zs.real)/np.pi*180.0,
                np.arctan2(cPZs.imag,cPZs.real)/np.pi*180.0,
                np.arctan2(c1Ps.imag,c1Ps.real)/np.pi*180.0,
                np.arctan2(c2Ps.imag,c2Ps.real)/np.pi*180.0,
                np.arctan2(c12s.imag,c12s.real)/np.pi*180.0]
        admit = [abs(c1Zs)/c11s,
                abs(c2Zs)/c22s,
                abs(cPZs)/cPPs,
                abs(c1Ps)/c11s,
                abs(c2Ps)/c22s,
                abs(c12s)/c11s]
        for i in range(6):
            for j in range(N):
                coh[i][j,:] = smooth(coh[i][j,:],40)
                admit[i][j,:] = smooth(admit[i][j,:],40)
        fig,axes = plt.subplots(6,1,sharex=True,figsize=[8,10.5])
        for title,i in zip(['1Z','2Z','PZ','1P','2P','12'],range(6)):
            cs = coh[i]
            axes[i].semilogx(np.tile(f,(len(cs),1)).T,cs.T,c=[0.5,0.5,0.5],lw=0.5)
            cs = cs[~isGood]
            axes[i].semilogx(np.tile(f,(len(cs),1)).T,cs.T,c=[1,0,0],lw=0.5)
            axes[i].set_ylim(0,1)
            axes[i].set_xlim(1e-4,max(f))
            axes[i].set_ylabel('Coherence')
            axes[i].set_title(f'{staID} Coherence: {title}')
        axes[-1].set_xlabel('Frequency (Hz)')
        plt.savefig(f'{dsetDir}/FIGURES/STATIONS_NOISEPROP/{staID}.coherence.png')
        plt.close()
        fig,axes = plt.subplots(6,1,sharex=True,figsize=[8,10.5])
        for title,i in zip(['1Z','2Z','PZ','1P','2P','12'],range(6)):
            cs = phase[i]
            axes[i].semilogx(np.tile(f,(len(cs),1)).T,cs.T,'o',c=[0.5,0.5,0.5],markersize=1)
            cs = cs[~isGood]
            axes[i].semilogx(np.tile(f,(len(cs),1)).T,cs.T,'o',c=[1,0,0],markersize=1)
            axes[i].set_ylim(-200,200)
            axes[i].set_xlim(1e-4,max(f))
            axes[i].set_ylabel('Phase')
            axes[i].set_title(f'{staID} Phase: {title}')
        axes[-1].set_xlabel('Frequency (Hz)')
        plt.savefig(f'{dsetDir}/FIGURES/STATIONS_NOISEPROP/{staID}.phase.png')
        plt.close()
        fig,axes = plt.subplots(6,1,sharex=True,figsize=[8,10.5])
        for title,i in zip(['1Z','2Z','PZ','1P','2P','12'],range(6)):
            cs = admit[i]
            axes[i].loglog(np.tile(f,(len(cs),1)).T,cs.T,c=[0.5,0.5,0.5],lw=0.5)
            cs = cs[~isGood]
            axes[i].loglog(np.tile(f,(len(cs),1)).T,cs.T,c=[1,0,0],lw=0.5)
            axes[i].set_xlim(1e-4,max(f))
            axes[i].set_ylabel('Admittance')
            axes[i].set_title(f'{staID} Admittance: {title}')
        axes[-1].set_xlabel('Frequency (Hz)')
        plt.savefig(f'{dsetDir}/FIGURES/STATIONS_NOISEPROP/{staID}.admittance.png')
        plt.close()

    if saveAvg:
        staID = f'{network}.{station}'
        os.makedirs(f'{dsetDir}/SPECTRA-AVG/{staID}',exist_ok=True)
        outfname = f'{staID}.spectraAvg.npz'
        np.savez_compressed(f'{dsetDir}/SPECTRA-AVG/{staID}/{outfname}',specprop=specprop)
        np.savetxt(f'{dsetDir}/SPECTRA-AVG/{staID}/isgood.txt',np.column_stack((specfiles,isGood)),fmt="%s")

    return specprop

def calTransfer(dsetDir,staID,tfLst=['ZP','Z1','Z2-1','ZP-21','ZH','ZP-H'],
                saveTransfer=True,overwrite=True,saveFigs=True,
                freqLimP=None,freqLimH=0.105,freqLimL=0.005):
    """ calculate transfer function and build basis function for decoupling
    Args:
        dsetDir: dataset directory to save results got in this module
        staID: {network name}.{station name}
        tfLst: list of transfer function you want to calculate
               e.g.: 'ZP-21' means transfer function from P to Z after remove
               component 1 and 2 successively
        saveTransfer: save transfer function or not
        saveFigs: save figures or not
    Saved in output
        tfLst: list of transfer function name, same as input
        tfs:   list of transfer function
        deNoiseBases: basis function prepaired for decoupling
        params: related parameters
    This function ......
    """
    if False:
        def fullTfLst(tfLst):
            newLst = []
            for tfStr in tfLst:
                tmp = tfStr.replace('-','')
                for i in range(len(tmp)-1):
                    newStr = tmp[0]+tmp[(-1-i):]
                    if len(newStr) > 2:
                        newStr = newStr[:2]+'-'+newStr[2:]
                    newLst.append(newStr)
            for i in range(len(newLst)-1,-1,-1):
                if newLst[i] in newLst[:i]:
                    del newLst[i]
            return newLst
        tfLst = fullTfLst(tfLst)
    if saveFigs:
        fig,axes = plt.subplots(len(tfLst),1,sharex=True,figsize=[8,10.5])
        isGoods = np.loadtxt(f'{dsetDir}/SPECTRA-AVG/{staID}/isgood.txt',dtype=str)
        if len(isGoods.shape) == 1:
            isGoods = [isGoods]
        isGoodDict = {}
        for isGood in isGoods:
            isGoodDict[isGood[0]] = isGood[1] == 'True'
    for specFile in glob.glob(f'{dsetDir}/SPECTRA/{staID}/*.npz'):
        os.makedirs(f'{dsetDir}/TRANSFUN/{staID}',exist_ok=True)
        outfname = f'{staID}.{specFile.split("/")[-1].split(".")[2]}.transfun.npz'
        outfpath = f'{dsetDir}/TRANSFUN/{staID}/{outfname}'
        if os.path.exists(outfpath):
            if overwrite:
                print(f'{outfpath} exists, overwrite it!')
            else:
                print(f'{outfpath} exists, skip it!')
                continue
        specprop = np.load(specFile,allow_pickle=True)['specprop'][()]
        ccDict = {'ZZ':specprop['power']['cZZ'],
                '11':specprop['power']['c11'],
                '22':specprop['power']['c22'],
                'PP':specprop['power']['cPP'],
                'HH':specprop['rotation']['cHH'],
                '1Z':specprop['cross']['c1Z'],
                '2Z':specprop['cross']['c2Z'],
                'PZ':specprop['cross']['cPZ'],
                'HZ':specprop['rotation']['cHZ'],
                '1P':specprop['cross']['c1P'],
                '2P':specprop['cross']['c2P'],
                'HP':specprop['rotation']['cHP'],
                '12':specprop['cross']['c12']}
        f = specprop['params']['f']
        dayID = specFile.split('/')[-1].split('.')[2]
        params = {'f':f,
                'staID':staID,
                'dayID':dayID,
                'elevation':specprop['params']['elev'],
                'oriAng':specprop['params']['rotor'],
                'NFFT':specprop['params']['NFFT'],
                'delta':specprop['params']['delta'],
                'nptsWin':specprop['params']['iptWinEs'][0] - specprop['params']['iptWinBs'][0] + 1}

        tfs = []
        deNoiseBases = []
        elev = -specprop['params']['elev']
        if freqLimP is None:
            freqLimP = np.sqrt(9.8/(2*np.pi*elev)) + 0.005
        def buildTaper(f,lo,hi):
            taper = np.ones(len(f))
            f1,f2,f3,f4 = lo*0.9,lo,hi,hi+0.01*max(f) #Helen
            # f1,f2,f3,f4 = lo-(hi-lo)/15.0,lo,hi,hi+(hi-lo)/15.0   # Ye
            taper[f<f1] = 0
            taper[f>f4] = 0
            f12 = f[(f>=f1)*(f<=f2)]
            taper[(f>=f1)*(f<=f2)] = np.cos(2*np.pi*(f12-f1)/(f2-f1))
            f34 = f[(f>=f3)*(f<=f4)]
            taper[(f>=f3)*(f<=f4)] = np.cos(2*np.pi*(f34-f3)/(f4-f3))
            return taper
        taperP = buildTaper(f,freqLimL,freqLimP)
        taperH = buildTaper(f,freqLimL,freqLimH)
        # freqLimL: low limit of frequency band pass when decouple
        # freqLimH: high limit for tilt
        # freqLimP: high limit for pressure/compliance
        def retriveCC(ccid, ccDict):
            if ccid in ccDict.keys():
                return ccDict[ccid]
            else:
                ccid = ccid[::-1]
                if ccid in ccDict.keys():
                    return ccDict[ccid].conj()
                else:
                    raise ValueError('Unknown cross-correlation ID!')
        def getTaper(comp1,comp2):
            compZ = ['Z']
            compH = ['1','2','H']
            compP = ['P']
            if (comp1 in compZ and comp2 in compH) or (comp2 in compZ and comp1 in compH):
                return taperH
            elif (comp1 in compH and comp2 in compH):
                return taperH
            elif (comp1 in compZ+compH and comp2 in compP) or (comp2 in compZ+compH and comp1 in compP):
                return taperP
            else:
                raise ValueError('Error in getTaper: unaccepted component pair')
        def getTF(tfcode,ccDict):
            tfcode = tfcode.replace('-','')
            if len(tfcode) == 4:
                z,c,b,a = tfcode
                taper1 = getTaper(c,b)+getTaper(z,b)-getTaper(c,b)*getTaper(z,b)
                taper2 = getTaper(c,b)*2-getTaper(c,b)**2
                x = getTF(z+c+a,ccDict) - getTF(z+b+a,ccDict)*getTF(b+c+a,ccDict)*taper1
                y = 1-getTF(c+b+a,ccDict)*getTF(b+c+a,ccDict)*taper2
                return x/y
            elif len(tfcode) == 3:
                z,b,a = tfcode
                taper1 = getTaper(b,a)+getTaper(z,a)-getTaper(b,a)*getTaper(z,a)
                taper2 = getTaper(b,a)*2-getTaper(b,a)**2
                x = getTF(z+b,ccDict) - getTF(z+a,ccDict)*getTF(a+b,ccDict)*taper1
                y = 1-getTF(a+b,ccDict)*getTF(b+a,ccDict)*taper2
                # if np.any(x==0) or np.any(y==0):
                #     print(tfcode)
                return x/y
            elif len(tfcode) == 2:
                return retriveCC(tfcode,ccDict) / retriveCC(tfcode[1]*2,ccDict)
            else:
                raise ValueError('Not supported yet!')
        def getdeNoiseBase(tfcode,ccDict):
            tfcode = tfcode.replace('-','')
            tfFunc = getTF(tfcode,ccDict)
            taper  = getTaper(tfcode[0],tfcode[1])
            compList = ['Z','1','2','P','H']
            if len(tfcode) == 4:
                z,c,b,a = tfcode
                return getdeNoiseBase(z+b+a,ccDict) - taper*tfFunc*getdeNoiseBase(c+b+a,ccDict)
            if len(tfcode) == 3:
                z,b,a = tfcode
                return getdeNoiseBase(z+a,ccDict) - taper*tfFunc*getdeNoiseBase(b+a,ccDict)
            if len(tfcode) == 2:
                base = np.zeros( [5,len(tfFunc)], dtype='complex')
                base[compList.index(tfcode[0])] = 1
                base[compList.index(tfcode[1])] = -taper*tfFunc
                return base
        for tfcode in tfLst:
            tfs.append(getTF(tfcode,ccDict))
            deNoiseBases.append(getdeNoiseBase(tfcode,ccDict))
        if saveTransfer:
            os.makedirs(f'{dsetDir}/TRANSFUN/{staID}',exist_ok=True)
            np.savez_compressed(f'{dsetDir}/TRANSFUN/{staID}/{outfname}',
                                tfLst=tfLst,tfs=tfs,deNoiseBases=deNoiseBases,params=params)

        if saveFigs:
            if isGoodDict[specFile]:
                c = [0.5,0.5,0.5]
            else:
                c = [1,0,0]
            for i in range(len(tfLst)):
                axes[i].loglog(f,abs(tfs[i]),c=c,lw=0.5)
                axes[i].set_xlim(1e-4,max(f))
                # axes[i].set_ylim(ylim[i][0],ylim[i][1])
                axes[i].set_title(f'{staID} Transfer Function {tfLst[i]}')
    os.makedirs(f'{dsetDir}/FIGURES/STATIONS_TRANSFUNC',exist_ok=True)
    plt.savefig(f'{dsetDir}/FIGURES/STATIONS_TRANSFUNC/{staID}.transfunc.png')
    plt.close()

def deCouple(trZ,tr1,tr2,trP,dsetDir,staID,evtID,evtTime,saveTraces=True,saveFigs=True,
    trType='disp',prefilt=None):
    """ decouple component based on previous calculated basis functions
    Args:
        trZ,tr1,tr2,trP: traces of Z, two horizontal and P component
        dsetDir:         directory to save all results got in this module
        staID: {network name}.{station name}
        evtID: event id, used to save decoupled records
        evtTime: event origin time, used to find basis functions from a day near it
        saveTraces: save all decoupled records or not
        saveFigs: save figures or not
    Returns:
        trCorrected: list of traces decoupled
    """
    # load event record
    if type(trZ)==str:
        trZ,tr1,tr2,trP = [obspy.read(trZ)[0],obspy.read(tr1)[0],
                           obspy.read(tr2)[0],obspy.read(trP)[0]]

    if prefilt is not None:
        for tr in [trZ,tr1,tr2,trP]:
            tr.detrend('linear')
            tr.taper(0.1,max_length=500)
            tr.filter('bandpass',freqmin=prefilt[0],freqmax=prefilt[1],zerophase=True)

    if trType == 'disp':
        pass
    elif trType == 'vel':
        # trZ.integrate()
        # tr1.integrate()
        # tr2.integrate()
        trZ = trIntegrate(trZ)
        tr1 = trIntegrate(tr1)
        tr2 = trIntegrate(tr2)
    else:
        raise ValueError(f'wrong trType:{trType}')
    # check traces
    for tr in [trZ,tr1,tr2,trP]:
        if tr.stats.starttime != trZ.stats.starttime:   # check starttime
            print('Different starttime found!')
            return False
        if tr.stats.delta != trZ.stats.delta:           # check sample rate
            print('Different sample rate found!')
            return False
        if tr.stats.npts != trZ.stats.npts:             # check npts
            print('Different npts found!')
            return False
        if np.all(tr.data == 0):                        # check if all-zero record
            print('All-zero record!')
            return False

    trZOri = trZ.copy()

    # get property of event record
    dt      = trZ.stats.delta
    npts    = trZ.stats.npts
    NFFT    = nextpow2(npts)
    fs      = 1.0/dt
    f       = fs/2*np.linspace(0,1,NFFT//2+1)

    # calculate specs, add zeros at both ends before fft
    specs = []
    npad0 = (NFFT - npts)//2
    for tr in [trZ,tr1,tr2,trP]:
        data = tr.data
        data = data * flatHann(len(data),0.075)
        data = data - data.mean()
        datapad = np.zeros(NFFT)
        datapad[npad0:npad0+npts] = data
        spec = np.fft.fft(datapad,NFFT)*dt
        spec = spec[:NFFT//2+1]
        specs.append(spec)

    # find noise record, closest to event and prefer noise record before event
    evtTime = UTCDateTime(evtTime)
    transFiles = glob.glob(f'{dsetDir}/TRANSFUN/{staID}/*.transfun.npz')
    transDates = [transFile.split('/')[-1].split('.')[2] for transFile in transFiles]
    tmp = abs(np.array([UTCDateTime(i)+43200-evtTime for i in transDates]))
    transFile = transFiles[np.argmin(tmp)]

    # load from transfer estimates
    tmp = np.load(transFile,allow_pickle=True)
    tfLst = list(tmp['tfLst'][()])
    deNoiseBases = tmp['deNoiseBases'][()]
    paramsBases = tmp['params'][()]
    fBases = paramsBases['f']

    # calculate H(rotated) component
    ang = paramsBases['oriAng']/180.0*np.pi
    specs.append( np.sin(ang)*specs[2]+np.cos(ang)*specs[1] )
    specs = np.array(specs)

    # interpolate to array f of event record
    deNoisedSpecs = []
    for deNoiseBase,tfCode in zip(deNoiseBases,tfLst):
        deNoiseBase = [np.interp(f,fBases,coef) for coef in deNoiseBase]
        spec = (specs*deNoiseBase).sum(axis=0)
        deNoisedSpecs.append(spec)

    trCorrected = []
    for deNoisedSpec in deNoisedSpecs:
        data = np.fft.ifft(2*deNoisedSpec,NFFT).real/dt
        tr = trZ.copy()
        tr.data = data[npad0:npad0+npts]
        trCorrected.append(tr)

    if saveTraces:
        for tr,tfCode in zip(trCorrected,tfLst):
            os.makedirs(f'{dsetDir}/CORRECTED/{staID}',exist_ok=True)
            tr.write(f'{dsetDir}/CORRECTED/{staID}/{staID}.{evtID}.{tfCode}.SAC',format='SAC')
        trZOri.write(f'{dsetDir}/CORRECTED/{staID}/{staID}.{evtID}.ori.SAC',format='SAC')

    if saveFigs:
        os.makedirs(f'{dsetDir}/FIGURES/CORRECTED',exist_ok=True)
        trLst = [trZ,tr1,tr2,trP]
        fig,axes = plt.subplots(len(trLst),1,sharex=True,figsize=[8,10.5])
        for i in range(len(trLst)):
            tr = trLst[i].copy()
            tr.filter('bandpass',freqmin=1.0/200.0,freqmax=1.0/10.0,corners=2,zerophase=True)
            axes[i].plot(tr.times(),tr.data,'k',lw=0.5)
            axes[i].set_xlim(tr.times()[0],tr.times()[-1])
            axes[i].set_title(('Z','H1','H2','P')[i])
        plt.savefig(f'{dsetDir}/FIGURES/CORRECTED/{staID}.{evtID}.origin.png')
        plt.close()

        fig,axes = plt.subplots(len(trCorrected),1,sharex=True,figsize=[8,10.5])
        for i in range(len(trCorrected)):
            tr = trCorrected[i].copy()
            tr.filter('bandpass',freqmin=1.0/200.0,freqmax=1.0/10.0,corners=2,zerophase=True)
            axes[i].plot(tr.times(),tr.data,'k',lw=0.5)
            axes[i].set_xlim(tr.times()[0],tr.times()[-1])
            axes[i].set_title(tfLst[i])
        plt.savefig(f'{dsetDir}/FIGURES/CORRECTED/{staID}.{evtID}.corrected.png')
        plt.close()

    return trCorrected

if __name__ == "__main__":
    plt.ioff()

    net = '7D'
    # sta = 'G30A'
    # chans = 'BHZ,BH1,BH2,BDH'
    sta = 'FN07A'
    chans = 'HHZ,HH1,HH2,HDH'

    from obspy.clients.fdsn import Client
    from obspy.core.util.attribdict import AttribDict
    def addSacHeader(st,inv):
        for tr in st:
            seedID = f'{tr.stats.network}.{tr.stats.station}.{tr.stats.location}.{tr.stats.channel}'
            tr.stats.sac = AttribDict()
            staCoord = inv.get_coordinates(seedID,tr.stats.starttime)
            tr.stats.sac.stel = staCoord['elevation']
            tr.stats.sac.stla = staCoord['latitude']
            tr.stats.sac.stlo = staCoord['longitude']

    client = Client('IRIS')
    inv = client.get_stations(network=net,station=sta,location='*',channel=chans,
                            level='channel')
    cata = client.get_events(starttime='2012-02-02',endtime='2012-02-03',minmagnitude=6.0)
    evtTime = cata[0].preferred_origin().time

    print('Downloading Noise ... ')
    stNoise = client.get_waveforms(network=net,station=sta,location='*',channel=chans,
                                starttime='2012-02-01',endtime='2012-02-02',attach_response=True)

    print('Downloading Event ... ')
    stEvent = client.get_waveforms(network=net,station=sta,location='*',channel=chans,
                                starttime=evtTime-1200,endtime=evtTime+7100,attach_response=True)

    print('Removing response ... ')
    for tr in stNoise+stEvent:
        if tr.stats.response.response_stages[0].input_units == 'PA':
            tr.remove_response(pre_filt=[0.001,0.002,1000,2000])
        elif tr.stats.response.response_stages[0].input_units in ('M/S','M'):
            tr.remove_response(pre_filt=[0.001,0.002,1000,2000], output='DISP')
        else:
            print(f'Unknown input unit, no response removed: {tr.get_id()}')

    print('Resampling ... ')
    stNoise = doResample(stNoise.copy(),0.2)
    stEvent = doResample(stEvent.copy(),0.2,round(7200.0/0.2))

    addSacHeader(stNoise,inv)

    print('Calculating Spectra ... ')
    specprop = calSpectra(stNoise[3],stNoise[1],stNoise[2],stNoise[0],dsetDir='NOISETC',
                        calrotation=True,segT=3600.0,saveFigs=True,saveSpec=True)

    print('Calculating Spectra - Average ... ')
    specpropAvg = cleanSpectra(dsetDir='NOISETC',staID=f'{net}.{sta}')

    print('Calculating Transfer Function ... ')
    calTransfer(dsetDir='NOISETC',staID=f'{net}.{sta}')

    print('Correcting ... ')
    trCorrected = deCouple(stEvent[3],stEvent[1],stEvent[2],stEvent[0],dsetDir='NOISETC',
                        staID=f'{net}.{sta}',evtID='201202021330',evtTime=evtTime)


import sys
import codecs

def main():
    with codecs.open(sys.argv[1], 'r', 'utf-8') as fin:
        with codecs.open(sys.argv[2], 'w', 'utf-8') as fout:
            for line in fin:
                words = line.strip().split()
                line_new = words[0] + '\t'
                for word in words[1:]:
                    for char in word:
                        if 'A' <= char <= 'Z':
                            line_new += char
                        else:
                            line_new += char + ' '
                    line_new += ' '
                line_new = line_new.replace('  ', ' ')
                fout.write('%s\n' % line_new)

if __name__ == '__main__':
    main()


import types
from pyomo.common.sorting import sorted_robust

def tostr(value, quote_str=False):
    """Convert a value to a string

    This function is a thin wrapper around `str(value)` to resolve a
    problematic __str__ implementation in the standard Python container
    types (tuple, list, and dict).  Those classes implement __str__ the
    same as __repr__ (by calling repr() on each contained object).  That
    is frequently undesirable, as you may wish the string representation
    of a container to contain the string representations of the
    contained objects.

    This function generates string representations for native Python
    containers (tuple, list, and dict) that contains the string
    representations of the contained objects.  In addition, it also
    applies the same special handling to any types that derive from the
    standard containers without overriding either __repn__ or __str__.

    Parameters
    ----------
    value: object
        the object to convert to a string
    quote_str: bool
        if True, and if `value` is a `str`, then return a "quoted
        string" (as generated by repr()).  This is primarily used when
        recursively processing native Python containers.

    Returns
    -------
    str

    """
    # Override the generation of str(list), but only if the object is
    # using the default implementation of list.__str__.  Note that the
    # default implemention of __str__ (in CPython) is to call __repr__,
    # so we will test both.  This is particularly important for
    # collections.namedtuple, which reimplements __repr__ but not
    # __str__.
    _type = type(value)
    if _type not in tostr.handlers:
        # Default to the None handler (just call str()), but override it
        # in particular instances:
        tostr.handlers[_type] = tostr.handlers[None]
        if isinstance(value, list):
            if ( _type.__str__ is list.__str__ and
                 _type.__repr__ is list.__repr__ ):
                tostr.handlers[_type] = tostr.handlers[list]
        elif isinstance(value, tuple):
            if ( _type.__str__ is tuple.__str__ and
                 _type.__repr__ is tuple.__repr__ ):
                tostr.handlers[_type] = tostr.handlers[tuple]
        elif isinstance(value, dict):
            if ( _type.__str__ is dict.__str__ and
                 _type.__repr__ is dict.__repr__ ):
                tostr.handlers[_type] = tostr.handlers[dict]
        elif isinstance(value, str):
            tostr.handlers[_type] = tostr.handlers[str]

    return tostr.handlers[_type](value, quote_str)

tostr.handlers = {
    list: lambda value, quote_str: (
        "[%s]" % (', '.join(tostr(v, True) for v in value))
    ),
    dict: lambda value, quote_str: (
        "{%s}" % (', '.join('%s: %s' % (tostr(k, True), tostr(v, True))
                            for k, v in value.items()))
    ),
    tuple: lambda value, quote_str: (
        "(%s,)" % (tostr(value[0], True),) if len(value) == 1
        else "(%s)" % (', '.join(tostr(v, True) for v in value))
    ),
    str: lambda value, quote_str: (
        repr(value) if quote_str else value
    ),
    None: lambda value, quote_str: str(value),
}

def tabular_writer(ostream, prefix, data, header, row_generator):
    """Output data in tabular form

    Parameters
    ----------
    ostream: io.TextIOBase
        the stream to write to
    prefix: str
        prefix each generated line with this string
    data: iterable
        an iterable object that returns (key, value) pairs
        (e.g., from iteritems()) defining each row in the table
    header: List[str]
        list of column headers
    row_generator: function
        a function that accepts the `key` and `value` from `data` and
        returns either a tuple defining the entries for a single row, or
        a generator that returns a sequence of table rows to be output
        for the specified `key`

    """

    prefix = tostr(prefix)

    _rows = {}
    # NB: _width is a list because we will change these values
    if header:
        header = (u"Key",) + tuple(tostr(x) for x in header)
        _width = [len(x) for x in header]
    else:
        _width = None
    _minWidth = 0

    for _key, _val in data:
        try:
            _rowSet = row_generator(_key, _val)
            if isinstance(_rowSet, types.GeneratorType):
                _rowSet = list(_rowSet)
            else:
                _rowSet = [_rowSet]
        except ValueError:
            # A ValueError can be raised when row_generator is called
            # (if it is a function), or when it is exhausted generating
            # the list (if it is a generator)
            _minWidth = 4 # Ensure columns are wide enough to output "None"
            _rows[_key] = None
            continue

        _rows[_key] = [
            ((tostr("" if i else _key),) if header else ())
            + tuple(tostr(x) for x in _r)
            for i, _r in enumerate(_rowSet) ]

        if not _rows[_key]:
            _minWidth = 4
        elif not _width:
            _width = [0]*len(_rows[_key][0])
        for _row in _rows[_key]:
            for col, x in enumerate(_row):
                _width[col] = max(_width[col], len(x), col and _minWidth)

    # NB: left-justify header entries
    if header:
        # Note: do not right-pad the last header with unnecessary spaces
        tmp = _width[-1]
        _width[-1] = 0
        ostream.write(prefix
                      + " : ".join( "%%-%ds" % _width[i] % x
                                    for i,x in enumerate(header) )
                      + "\n")
        _width[-1] = tmp

    # If there is no data, we are done...
    if not _rows:
        return

    # right-justify data, except for the last column if there are spaces
    # in the data (probably an expression or vector)
    _width = ["%"+str(i)+"s" for i in _width]

    if any( ' ' in r[-1]
            for x in _rows.values() if x is not None
            for r in x  ):
        _width[-1] = '%s'
    for _key in sorted_robust(_rows):
        _rowSet = _rows[_key]
        if not _rowSet:
            _rowSet = [ [_key] + [None]*(len(_width)-1) ]
        for _data in _rowSet:
            ostream.write(
                prefix
                + " : ".join( _width[i] % x for i,x in enumerate(_data) )
                + "\n")

class StreamIndenter(object):
    """
    Mock-up of a file-like object that wraps another file-like object
    and indents all data using the specified string before passing it to
    the underlying file.  Since this presents a full file interface,
    StreamIndenter objects may be arbitrarily nested.
    """

    def __init__(self, ostream, indent=' '*4):
        self.os = ostream
        self.indent = indent
        self.stripped_indent = indent.rstrip()
        self.newline = True

    def __getattr__(self, name):
        return getattr(self.os, name)

    def write(self, data):
        if not len(data):
            return
        lines = data.split('\n')
        if self.newline:
            if lines[0]:
                self.os.write(self.indent+lines[0])
            else:
                self.os.write(self.stripped_indent)
        else:
            self.os.write(lines[0])
        if len(lines) < 2:
            self.newline = False
            return
        for line in lines[1:-1]:
            if line:
                self.os.write("\n"+self.indent+line)
            else:
                self.os.write("\n"+self.stripped_indent)
        if lines[-1]:
            self.os.write("\n"+self.indent+lines[-1])
            self.newline = False
        else:
            self.os.write("\n")
            self.newline = True

    def writelines(self, sequence):
        for x in sequence:
            self.write(x)


# """Prime Number"""

# def prime_checker(number):
#     is_prime = True
#     for i in range (2, number):
#         if number % i == 0:
#             print('not prime')
#     if is_prime:
#         print('prime')
#     else:
#         print('not')

# n = int(input("Check this number: "))
# prime_checker(number=n)

"""Ceasar's Cipher"""

alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']

direction = input("Type 'encode' to encrypt, type 'decode' to decrypt:\n")
text = input("Type your message:\n").lower()
shift = int(input("Type the shift number:\n"))

def caesar(basic_text, position_shift, cipher_direction):

    caesar_cipher = ''
    position_shift = position_shift % 26
    if cipher_direction == 'decode':
        position_shift *= -1
    for letter in basic_text:
        if letter in alphabet:
            position = alphabet.index(letter)
            new_position = position + position_shift
            caesar_cipher += alphabet[new_position]
        else:
            caesar_cipher += letter
    print(f'the direction is: {cipher_direction}, and the message is {caesar_cipher}')

# caesar(basic_text = text, position_shift = shift, cipher_direction = direction)

end = False
while not end:
    direction = input("Type 'encode' to encrypt, type 'decode' to decrypt:\n")
    text = input("Type your message:\n").lower()
    shift = int(input("Type the shift number:\n"))

    shift = shift % 26
    caesar(basic_text = text, position_shift = shift, cipher_direction = direction)

    restart = print(input('Would you like to decode again??  Y/N '))

    if restart == "N":
        end = True
        print('Farewell')  #nadvor od looop so end isto nadvor od loop so while klauza

caesar(text, shift, direction)

# # def encrypt(basic_text, shift_ammount):
# #     ceasar_cipher = ''
# #     for letter in basic_text:
# #         position = alphabet.index(letter)
# #         new_position = position + shift_ammount
# #         # ciphered = alphabet[new_position]
# #         ceasar_cipher += alphabet[new_position]
# #     print(f'The encoded text is {ceasar_cipher}')

# # def decrypt(encrypt, shift_ammount):
# #     caesar_cipher = ''
# #     for letter in encrypt:
# #         position = alphabet.index(letter)
# #         old_position = position - shift_ammount
# #         caesar_cipher += alphabet[old_position]
# #     print(f'the decoded text is {caesar_cipher}')


from code_breaking_utils import *
from enigma import *

#   This is the basic Enigma brute force code breaking that runs in a single process
#
#   Use the CLI to try the code breaking:
#       python3 enigma-cli.py --module interactive
#   Select 2 for code breaking, then 1 for predefined demo job,
#   then 1 for the single process code breaker
#
#   Function decrypt_cipher finds Enigma configuration based on encrypted
#   text, crib and a partially known configuration:
#
#       decrypt_cipher(encrypted_text, crib, config_string, sample_size = 500)
#
#
#   Function decrypt_cipher_reflector_scrambled (special case related to my uni project
#   finds Enigma configuration based on encrypted text, crib and a partially
#   known configuration and knowing that two wires of the Reflector had been swapped.
#
#       decrypt_cipher_reflector_scrambled(encrypted_text, crib, enigma_config)
#

def decrypt_cipher(encrypted_text, crib, config_string, sample_size = 500):
    """Attempt to break Enigma cypher with a known crib and partially known config

    Example input:
       encrypted_text:
       ABSKJAKKMRITTNYURBJFWQGRSGNNYJSDRYLAPQWIAGKJYEPCTAGDCTHLCDRZRFZHKNRSDLNPFPEBVESHPY
       cribs: THOUSANDS
       enigma_config
       C III-?-["II","I"]-V [4,5,6]-24-?-7 ABGZ-?-Q-F AQ-?S-ED-["ZU","ZF","ZK"]

    To understand how to specify the partially known or unknown Enigma config
    read the description of the all_enigma_settings_candidates() function.

    After all possible Enigma settings are constructed each setting is tried on a crib
    until such Enigma setting is found that the complete crib matches the encoded text.
    Such Enigma setting is put on a list of possible candidates along with the decrypted
    text. A human must then go through this list to decide if any of the potential settings
    are correct.

    :param encrypted_text:
    :param crib:
    :param config_string:
    :param sample_size: size of the sample to predict remaining time from
    :return:
    """
    time_start = time.time()

    if encrypted_text is None or len(encrypted_text) < len(crib):
        raise ValueError('Expected some code to break.')
    if not crib:
        raise ValueError('Expected a crib.')

    # Construct all possible Enigma settings based on unknown / partially known
    # Enigma configuration provided:
    enigma_configs = all_possible_settings(config_string)
    # Slide the crib under the encrypted text and determine all positions in
    # which the letter of the crib does not match the encrypted text
    # (Enigma can never encode a letter into itself)
    crib_positions = possible_crib_positions(encrypted_text, crib)

    print("\nRunning a single process to find solutions.")
    print("{0} settings to search".format(len(enigma_configs)*len(crib_positions)))

    # Loop through all potential Enigma settings and try to decrypt the crib:
    configs_to_check = []
    for pos in crib_positions:
        for cnf in enigma_configs:
            configs_to_check.append((cnf,pos))

    result = check_enigma_config(configs_to_check,
                                 crib,
                                 encrypted_text,
                                 sample_size)

    return result, time.time() - time_start

# ------------------------------------------------------------------------
#           Scrambled Reflector Case
#
# All methods below this line are for breaking a specific case
# which is imaginary, where an Enigma reflector had it's
# connections scrambled. So a reflector has been meddled with
# by swapping two wires (4 pairs changed)
# ------------------------------------------------------------------------

def decrypt_cipher_reflector_scrambled(encrypted_text, crib, enigma_config):
    """Attempt to break Enigma cypher with a known crib and a reflector that
    had been hacked (2 wires in the reflector were swapped)

    Example input:
       encrypted_text:
       HWREISXLGTTBYVXRCWWJAKZDTVZWKBDJPVQYNEQIOTIFX
       cribs: INSTAGRAM
       enigma_config
       "? V-II-IV 6-18-7 A-J-L UG-IE-PO-NX-WT"

    To understand how to specify the partially known or unknown Enigma config
    read the description of the all_enigma_settings_candidates() function.

    After all possible Enigma settings are constructed additionally all possible
    permutations of every possible reflector is added. Every reflector has been
    meddled with by swapping two wires (4 pairs changed). Details on how these
    permutations were constructed in docstring here:
        permutate_reflector_by_wire_swap()

    Every Enigma setting with every possible reflector permutation will be tried
    until the encoded crib matches the encoded text. Such Enigma setting is put
    on a list of possible candidates along with the decrypted text. A human must
    then go through this list to decide if any of the potential settings are
    correct.

    :param encrypted_text:
    :param crib:
    :param config_string:
    :return:
    """
    time_start = time.time()

    crib_positions = possible_crib_positions(encrypted_text, crib)
    enigma_configs = all_possible_settings(enigma_config)

    print("\nRunning a single process to find solutions")
    print("\nReflector has two wires swapped.")
    print("{0} settings (4290 reflector wirings) to search".format(len(enigma_configs) * 4290 * len(crib_positions)))

    potential_configs = []
    for cnf in enigma_configs:
        enigma_instance = Enigma(cnf)
        l = permutate_reflector_by_wire_swap("".join(enigma_instance.rotors[-1].left_pins), 2)
        #print("Checking enigma config={0} with {1} reflector wirings".format(enigma_instance.print_state(), len(l)))
        for pos in crib_positions:
            for reflector_option in l:
                enigma_instance = Enigma(cnf)
                # Replace standard reflector with a hacked one:
                enigma_instance.rotors[-1].left_pins = reflector_option
                enigma_instance.rotate_n_steps(pos)
                potential_config = True
                for inx in range(len(crib)):
                    if enigma_instance.encode_character(crib[inx]) != encrypted_text[pos + inx]:
                        potential_config = False
                        break
                if potential_config:
                    e = Enigma(cnf)
                    e.rotors[-1].left_pins = reflector_option
                    potential_configs.append((str(cnf), e.encode_string(encrypted_text), reflector_option))
    return potential_configs, time.time() - time_start


def busquedaStirng(itemSearch, TuplaItems):
    for item in TuplaItems:
        if itemSearch in item:
            return "Si esta en tus contactos"
        elif itemSearch in item[0]:
            return "Si esta en tus contactos"
        else:
            return "No esta entre tus contactos"

def busquedaTupla(itemSearch, TuplaItems):
    for item in TuplaItems:
        if itemSearch == item:
            return "Si esta en tus contactos"
        else:
            return "No esta entre tus contactos"

def tipoSearch(argument):
    lista = (("Rabbit Rabbit", "000000000"), ("persona aleatoria", "675832376"), ("Mefisto infernal", "666666666"))
    switch = {
        1: conTodo,
        2: nombreCompleto,
        3: soloNombre,
        4: soloApellido,
        5: soloTlf
    }
    ver = switch.get(int(argument), lambda: "Invalid number")
    ver(lista)

def nombreCompleto(lista):
    for elem in lista:
        print(elem[0] + " --- " + elem[1])
    str = input("\nnombre\n") + " " + input("\napellido\n")
    item = (str)

    print(busquedaStirng(item, lista))

def conTodo(lista):
    for elem in lista:
        print(elem[0] + " --- " + elem[1])
    str = input("\nnombre\n") + " " + input("\napellido\n")
    item = (str, input("telefono\n"))

    print(busquedaTupla(item, lista))

def soloNombre(lista):
    for elem in lista:
        print(elem[0] + " --- " + elem[1])
    item = (input("\nnombre\n"))

    print(busquedaStirng(item, lista))

def soloApellido(lista):
    for elem in lista:
        print(elem[0] + " --- " + elem[1])
    item = (input("\napellido\n"))

    print(busquedaStirng(item, lista))

def soloTlf(lista):
    for elem in lista:
        print(elem[0] + " --- " + elem[1])
    item = (input("telefono\n"))

    print(busquedaStirng(item, lista))

tipoSearch(input("""elije
    1: conTodo,
    2: nombreCompleto,
    3: soloNombre,
    4: soloApellido,
    5: soloTlf
    """))


import numpy as np
import matplotlib.pyplot as plt

from PIL import Image, ImageDraw, ImageFont
from libPlotter.testVideoHelper import TestVideoHelper

#
# to plot results of a Donkey Car dataset, with angles and throttle as floats
#
class TestResultsPlotter2(object) :

    @staticmethod
    def mark_image(image, angle, color_angle=0):

        img_width = image.shape[1]
        img_height = image.shape[0]

        video_img = Image.fromarray(image)
        pdraw = ImageDraw.Draw(video_img)

        # pdraw.text((1, 0), "{:.3f}".format(angle), color_angle, font=ImageFont.truetype("arial", 9))

        # draw a line in the middle, tilted to show wheels angle:
        angle = TestVideoHelper.clamp(angle, -1, 1)  # should be within the range anyway
        margin_top = 3
        margin_bottom = 1
        l = img_height - (margin_top + margin_bottom)
        dx = l * np.sin(angle)
        x = img_width / 2 + dx

        line_width = 1

        pdraw.line((x, margin_top, img_width / 2, img_height - margin_bottom),
                   fill=color_angle, width=line_width)

        return np.asarray(video_img, dtype=np.float32)

    @staticmethod
    def plot_image(i, predictions_array, img, test_angles, test_throttles):
        predictions_array, img, true_angle = predictions_array[i], img[i], test_angles[i]
        plt.grid(False)
        plt.xticks([])
        plt.yticks([])

        size_th = img.shape[0]

        predicted_angle = np.argmax(predictions_array)

        video_img = TestResultsPlotter2.mark_image(img.reshape((size_th, size_th)),
                                                   true_angle, color_angle=960)
        video_img = TestResultsPlotter2.mark_image(video_img,
                                                   predicted_angle, color_angle=180)
        plt.imshow(video_img, cmap=plt.cm.binary)

        if predicted_angle == true_angle:
            color = 'blue'
        else:
            color = 'red'

        max_pred = 100 * np.max(predictions_array)
        plt.xlabel("{:1.3f} ({:1.3f})".format(predicted_angle, true_angle), color=color)

    @staticmethod
    def plot_value_array(i, predictions_array, test_angles, test_throttles):
        predictions_array, true_label = predictions_array[i], test_angles[i]
        plt.grid(False)
        plt.xticks([])
        plt.yticks([])
        thisplot = plt.bar(range(50), predictions_array, color="#777777")
        plt.ylim([0, 1])
        predicted_label = np.argmax(predictions_array)

        #thisplot[predicted_label].set_color('red')
        #thisplot[true_label].set_color('blue')

    @staticmethod
    def plotBefore(out_filename, train_images, train_angles, test_throttles):
        num_rows = 50
        num_cols = 8
        line_width = 1  # of the elements in the large image
        color_angle = 96
        # color_throttle = 255

        num_images = num_rows * num_cols
        fig = plt.figure(figsize=(2 * num_cols, 2 * num_rows))
        for i in range(num_images):
            plt.subplot(num_rows, num_cols, i + 1)
            plt.xticks([])
            plt.yticks([])
            plt.grid(False)

            size_th = train_images[i].shape[0]
            angle = train_angles[i]
            video_img = TestResultsPlotter2.mark_image(train_images[i].reshape((size_th, size_th)), angle)

            plt.imshow(video_img, cmap=plt.cm.binary)
            plt.xlabel("{:.3f}".format(angle))

        fig.savefig(out_filename, bbox_inches='tight')

    @staticmethod
    def plotResults(out_filename, predictions, test_images, test_angles, test_throttles):
        # Plot the first X test images, their predicted label, and the true label
        # Color correct predictions in blue, incorrect predictions in red
        num_rows = 50
        num_cols = 4
        num_images = num_rows * num_cols
        fig=plt.figure(figsize=(2 * 2 * num_cols, 2 * num_rows))
        for i in range(num_images):
            plt.subplot(num_rows, 2 * num_cols, 2 * i + 1)
            TestResultsPlotter2.plot_image(i, predictions, test_images, test_angles, test_throttles)
            plt.subplot(num_rows, 2 * num_cols, 2 * i + 2)
            TestResultsPlotter2.plot_value_array(i, predictions, test_angles, test_throttles)

        #plt.show()
        fig.savefig(out_filename,bbox_inches='tight')


import heapq

input_tile = []
f = open("input_day15.txt", "r")
for line in f:
    input_tile.append(list(map(int, line.strip())))
f.close()
input_row_size = len(input_tile)
input_col_size = len(input_tile[0])

cavern = [list() for _ in range(input_row_size * 5)]

def tile_step(tile, step):
    mutated_tile = []
    for row in tile:
        new_row = []
        for val in row:
            new_val = val + 1 * (step % 9)
            if new_val > 9:
                new_val -= 9
            new_row.append(new_val)
        mutated_tile.append(new_row)
    return mutated_tile

tile_idx = 0
while tile_idx < 25:
    tile_row = (tile_idx // 5) * input_row_size
    new_tile = tile_step(input_tile, tile_idx // 5 + tile_idx % 5)
    for row_idx, row in enumerate(new_tile):
        cavern[tile_row + row_idx].extend(row)
    tile_idx += 1

cavern_width, cavern_height = len(cavern), len(cavern[0])
prio_q = [(cavern[0][0], (0, 0))]
dirs = [(1, 0), (-1, 0), (0, 1), (0, -1)]
memo = [[float("inf")] * cavern_width for _ in range(cavern_height)]
memo[0][0] = cavern[0][0]

def can_go(row_idx, col_idx):
    return (
        row_idx >= 0
        and row_idx < cavern_height
        and col_idx >= 0
        and col_idx < cavern_width
    )

while prio_q:
    origin = heapq.heappop(prio_q)
    origin_val = memo[origin[1][0]][origin[1][1]]
    for dir in dirs:
        visit_row, visit_col = origin[1][0] + dir[0], origin[1][1] + dir[1]
        if can_go(visit_row, visit_col):
            test_sum = origin_val + cavern[visit_row][visit_col]
            if test_sum < memo[visit_row][visit_col]:
                memo[visit_row][visit_col] = test_sum
                heapq.heappush(prio_q, (test_sum, (visit_row, visit_col)))
print(memo[-1][-1] - memo[0][0])


from django.urls import path
from .views import getAllBlogs, getBlogById, createBlog, updateBlog, deleteBlog, register, getBlogByUser, likeBlog, updateBlogAdmin
urlpatterns = [
    path('register/', register, name='register'),
    path('blogs/', getAllBlogs, name='blogs'),
    path('blogs/<int:pk>', getBlogById, name='blog'),
    path('user/blogs/', getBlogByUser, name='user-blogs'),
    path('blog/', createBlog, name='create-blog'),
    path('blog/<int:pk>', updateBlog, name='update-blog'),
    path('edit-blog/<int:pk>', updateBlogAdmin, name='update-blog'),
    path('delete/<int:pk>', deleteBlog, name='delete-blog'),
    path('like/<int:pk>', likeBlog, name='like-blog'),
]