from glad.loader import BaseLoader
from glad.loader.c import LOAD_OPENGL_DLL, LOAD_OPENGL_DLL_H

_GLX_LOADER = \
    LOAD_OPENGL_DLL % {'pre':'static', 'init':'open_gl',
                       'proc':'get_proc', 'terminate':'close_gl'} + '''
int gladLoadGLX(void) {
    if(open_gl()) {
        gladLoadGLXLoader((LOADER)get_proc);
        close_gl();
        return 1;
    }

    return 0;
}
'''

_GLX_HEADER = '''
#include <X11/X.h>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <glad/glad.h>

#ifndef __glad_glxext_h_

#ifdef __glxext_h_
#error GLX header already included, remove this include, glad already provides it
#endif

#define __glad_glxext_h_
#define __glxext_h_

#if defined(_WIN32) && !defined(APIENTRY) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#include <windows.h>
#endif

#ifndef APIENTRY
#define APIENTRY
#endif
#ifndef APIENTRYP
#define APIENTRYP APIENTRY *
#endif
#ifndef GLAPI
#define GLAPI extern
#endif

#ifdef __cplusplus
extern "C" {
#endif

typedef void* (* LOADER)(const char *name);
void gladLoadGLXLoader(LOADER);
'''

_GLX_HEADER_LOADER = '''
int gladLoadGLX(void);
''' + LOAD_OPENGL_DLL_H

_GLX_HEADER_END = '''
#ifdef __cplusplus
}
#endif

#endif
'''

_GLX_HAS_EXT = '''
'''

class GLXCLoader(BaseLoader):
    def write(self, fobj, apis):
        if not self.disabled:
            fobj.write(_GLX_LOADER)

    def write_begin_load(self, fobj):
        pass

    def write_find_core(self, fobj):
        pass

    def write_has_ext(self, fobj):
        fobj.write(_GLX_HAS_EXT)

    def write_header(self, fobj):
        fobj.write(_GLX_HEADER)
        if not self.disabled:
            fobj.write(_GLX_HEADER_LOADER)

    def write_header_end(self, fobj):
        fobj.write(_GLX_HEADER_END)


"""
Created on Thu Mar 16 04:12:40 2017

@author: Daniel Banco
"""

import numpy as np
from numpy.linalg import norm

import DataAnalysis as DA
import EllipticModels as EM
import LassoSolvers
import RingImageProcessing as RingIP
import CirculantOperations as CO
import os
import time

class RingModel:
    def __init__(self, load_step, img_num, ring_radius_low, ring_radius_high,
				   num_theta, num_rad, dtheta, drad, var_theta, var_rad):

        # Metadata
        self.load_step = load_step
        self.img_num = img_num
        self.ring_radius_low = ring_radius_low
        self.ring_radius_high = ring_radius_high

        # Domain
        self.num_theta = num_theta
        self.num_rad = num_rad
        self.dtheta = dtheta
        self.drad = drad
        self.var_theta = var_theta
        self.var_rad = var_rad

    def compute_polar_image(self, sample):
        image = sample.load_image(self.img_num, self.load_step)

        polar_image = np.zeros((self.num_rad, self.num_theta))

        for i, r in enumerate(np.arange(self.ring_radius_low,self.ring_radius_high,1)):
            fi, eta_domain = RingIP.azimuthal_projection(image,
												                sample.true_center,
                                                         r,
                                                         0,
                                                         2*np.pi-self.dtheta,
                                                         self.num_theta)
            polar_image[i,:] = fi

        self.polar_image = polar_image

    def generate_A0ft_svd_list(self):
        A0ft_list = EM.unshifted_basis_ft_svd_list(self.var_theta,
									                               self.var_rad,
									                               self.dtheta,
									                               self.drad,
									                               self.num_theta,
									                               self.num_rad)
        return A0ft_list

    def compute_lipschitz(self,A0_stack):
        lipschitz = 0
        for i in range(self.var_theta.shape[0]):
            for j in range(self.var_rad.shape[0]):
                lipschitz += max(np.linalg.eig( A0_stack[:,:,i,j] ).real)
        print(lipschitz)
        print(lipschitz*self.num_rad*self.num_theta)
        self.lipschitz = lipschitz

    def fit_circulant_FISTA(self,A0ft_stack,out_path,positive=1,benchmark=0,verbose=0):
        x_hat, times = LassoSolvers.fista_circulant_2D(A0ft_stack, self.polar_image, self.lipschitz, self.l1_ratio, self.max_iters,
                         positive=positive,
											   benchmark=benchmark,
											   verbose=verbose)

        y_hat = CO.Ax_ft_2D(A0ft_stack,x_hat)

        self.fit_image = y_hat
        self.times = times
        self.coefs = x_hat
        self.fit_error = norm(y_hat-self.polar_image)
        self.rel_fit_error = self.fit_error/norm(self.polar_image)

        np.save(os.path.join(out_path,'_load_',str(self.load_step),'_img_',str(self.img_num),'.npy'),self)

        return self

    def fit_circulant_svd_FISTA(self,A0ft_svd_list,positive=1,benchmark=0,verbose=0):
        x_hat, times = LassoSolvers.fista_circulant_2D_SVD(A0ft_svd_list,len(self.var_theta), len(self.var_rad), self.polar_image,self.lipschitz, self.l1_ratio, self.max_iters,
                         positive=positive,
											   benchmark=benchmark,
											   verbose=verbose)

        y_hat = LassoSolvers.Ax_ft_2D_SVD(A0ft_svd_list,x_hat)

        self.fit_image = y_hat
        self.times = times
        self.coefs = x_hat
        self.fit_error = norm(y_hat-self.polar_image)
        self.rel_fit_error = self.fit_error/norm(self.polar_image)

    def fit_parallel_circulant_FISTA(self,A0ft_svd_list,positive=1,benchmark=0,verbose=0):
        x_hat, times = LassoSolvers.fista_circulant_2D_Parallel_SVD(A0ft_svd_list,len(self.var_theta), len(self.var_rad), self.polar_image,
 self.lipschitz, self.l1_ratio, self.max_iters,   positive=positive,

           benchmark=benchmark,

           verbose=verbose)

        y_hat = CO.Ax_ft_2D_Parallel_SVD(A0ft_svd_list,x_hat)

        self.fit_image = y_hat
        self.times = times
        self.coefs = x_hat
        self.fit_error = norm(y_hat-self.polar_image)
        self.rel_fit_error = self.fit_error/norm(self.polar_image)

    def print_fit_stats(self):

        print('Relative Error: '  + str(rel_fit_error))

        pos_coef = np.sum(self.coefs>0)
        tot_coef = len(self.coefs.ravel())
        sparsity0 = pos_coef/tot_coef
        one_coef = np.sum(self.coefs>1)
        sparsity1 = one_coef/tot_coef

        print('x > 0         |  x > 1             | total x')
        print(str(pos_coef) + '       |  ' + str(one_coef) + '             | ' + str(tot_coef))
        print(str(sparsity0) + ' |  ' + str(sparsity1))

def compute_lipschitz(ringModel,A0_stack):
    lipschitz = 0
    for i in range(ringModel.var_theta.shape[0]):
        for j in range(ringModel.var_rad.shape[0]):
            eigs,vecs = np.linalg.eig( np.dot(A0_stack[:,:,i,j],A0_stack[:,:,i,j].T) )
            lipschitz += np.max(eigs.real)
    print(lipschitz)
    print(lipschitz*ringModel.num_rad*ringModel.num_theta)
    ringModel.lipschitz = lipschitz

def generate_A0_stack(ringModel):
    A0_stack = EM.unshifted_basis_matrix_stack(ringModel.var_theta,
							                               ringModel.var_rad,
							                               ringModel.dtheta,
							                               ringModel.drad,
							                               ringModel.num_theta,
							                               ringModel.num_rad)
    return A0_stack

def fit_circulant_FISTA_Multiprocess(ringModel,A0ft_stack,out_path,positive=1,benchmark=0,verbose=0):
    x_hat, times = LassoSolvers.fista_circulant_2D(A0ft_stack, ringModel.polar_image, ringModel.lipschitz, ringModel.l1_ratio, ringModel.max_iters,
                     positive=positive,
									   benchmark=benchmark,
									   verbose=verbose)

    y_hat = CO.Ax_ft_2D(A0ft_stack,x_hat)

    ringModel.fit_image = y_hat
    ringModel.times = times
    ringModel.coefs = x_hat
    ringModel.fit_error = norm(y_hat-ringModel.polar_image)
    ringModel.rel_fit_error = ringModel.fit_error/norm(ringModel.polar_image)

    np.save(out_path+'_load_'+str(ringModel.load_step)+'_img_'+str(ringModel.img_num)+'.npy',ringModel)


import os
import time
#message = "autobackup " + time.strftime("%d-%m-%Y %H:%M:%S", time.localtime())
commands = [
    "sudo service nginx stop | tee ../log.txt",
    "sudo supervisorctl stop uacr | tee -a ../log.txt",
    "sudo git fetch --all | tee -a ../log.txt",
    "sudo git reset --hard origin/master | tee -a ../log.txt",
    "sudo supervisorctl start uacr | tee -a ../log.txt",
    "sudo service nginx start | tee -a ../log.txt",
]

try:
    for command in commands:
        #print (command)
        os.system(command)
        time.sleep(1)

except:
    print("ERROR!")
else:
    print("DONE!!")


import random
import traceback

import discord
from discord.ext import commands
from requests import HTTPError

from poediscordbot.cogs.pob import util
from poediscordbot.cogs.pob.output import pob_output
from poediscordbot.cogs.pob.poe_data import poe_consts
from poediscordbot.cogs.pob.util import pastebin
from poediscordbot.pob_xml_parser import pob_xml_parser
from poediscordbot.util.logging import log

def setup(bot, active_channels, allow_pming=True):
    bot.add_cog(PoBCog(bot, active_channels, allow_pming))

class PoBCog(commands.Cog):
    def __init__(self, bot, active_channels, allow_pming=True):
        self.bot = bot
        self.active_channels = active_channels
        self.allow_pming = allow_pming

    @commands.Cog.listener()
    async def on_message(self, message):
        """
        Handle message events
        :param message: received
        :return: None
        """
        react_to_dms = isinstance(message.channel, discord.abc.PrivateChannel) and self.allow_pming

        if message.author.bot:
            return

        if 'help' in message.content.lower() and react_to_dms:
            await message.channel.send("Paste your pastebin here for a quick overview or use '!pob <pastebin>' for a "
                                       "detailed response.")
            return

        if (react_to_dms or message.channel.name in self.active_channels) \
                and not util.starts_with("!pob", message.content[:4]) \
                and "pastebin.com/" in message.content:
            # check if valid xml
            # send message
            log.debug(f"A| {message.channel}: {message.content}")
            try:
                embed = self._parse_pob(message.author, message.content, minify=True)
                if embed:
                    await message.channel.send(embed=embed)
            except HTTPError as err:
                log.error(f"Pastebin: Invalid pastebin-url msg={err}")
            except pastebin.CaptchaError as err:
                log.error(f"Pastebin: Marked as spam msg={err}")
                await message.channel.send(err.message)

    @commands.command(pass_context=True)
    async def pob(self, ctx, *, key):
        if not self.allow_pming and ctx.message.channel.is_private:
            return
        embed = self._parse_pob(ctx.message.author, ctx.message.content)
        try:
            if embed:
                await ctx.message.channel.send(embed=embed)
        except discord.Forbidden:
            log.info("Tried pasting in channel without access.")

    @staticmethod
    def _parse_pob(author, content, minify=False):
        """
        Trigger the parsing of the pastebin link, pass it to the output creating object and send a message back
        :param minify: show minified version of the embed
        :param content: of the message
        :param author: user sending the message
        :return: Embed
        """
        paste_keys = pastebin.fetch_paste_key(content)
        if paste_keys:
            paste_key = random.choice(paste_keys)
            log.info(f"Parsing pastebin with key={paste_key} from author={author}")
            raw_data = pastebin.get_as_xml(paste_key)
            if not raw_data:
                log.error(f"Unable to obtain raw data for pastebin with key {paste_key}")
                return

            xml = pastebin.decode_to_xml(raw_data)
            if not xml:
                log.error(f"Unable to obtain xml data for pastebin with key {paste_key}")
                return

            web_poe_token = util.fetch_xyz_pob_token(raw_data)
            build = pob_xml_parser.parse_build(xml)
            try:
                embed = pob_output.generate_response(author, build, minified=minify, pastebin_key=paste_key,
                                                     non_dps_skills=poe_consts, web_poe_token=web_poe_token)
                log.debug(f"embed={embed}; thumbnail={embed.thumbnail}")
                return embed
            except Exception as e:
                ex_msg = ''.join(traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__))
                log.error(f"Could not parse pastebin={paste_key} - Exception={ex_msg}")


import os
import numpy as np
import multiprocessing
from functools import partial
from keras.preprocessing.image import Iterator, _count_valid_files_in_directory, _list_valid_filenames_in_directory, \
    load_img, img_to_array
import keras.backend as K

class DirectoryIteratorWithFname(Iterator):
    """Iterator capable of reading images from a directory on disk.
    # Arguments
        directory: Path to the directory to read images from.
            Each subdirectory in this directory will be
            considered to contain images from one class,
            or alternatively you could specify class subdirectories
            via the `classes` argument.
        image_data_generator: Instance of `ImageDataGenerator`
            to use for random transformations and normalization.
        target_size: tuple of integers, dimensions to resize input images to.
        color_mode: One of `"rgb"`, `"grayscale"`. Color mode to read images.
        classes: Optional list of strings, names of sudirectories
            containing images from each class (e.g. `["dogs", "cats"]`).
            It will be computed automatically if not set.
        class_mode: Mode for yielding the targets:
            `"binary"`: binary targets (if there are only two classes),
            `"categorical"`: categorical targets,
            `"sparse"`: integer targets,
            `"input"`: targets are images identical to input images (mainly
                used to work with autoencoders),
            `None`: no targets get yielded (only input images are yielded).
        batch_size: Integer, size of a batch.
        shuffle: Boolean, whether to shuffle the data between epochs.
        seed: Random seed for data shuffling.
        data_format: String, one of `channels_first`, `channels_last`.
        save_to_dir: Optional directory where to save the pictures
            being yielded, in a viewable format. This is useful
            for visualizing the random transformations being
            applied, for debugging purposes.
        save_prefix: String prefix to use for saving sample
            images (if `save_to_dir` is set).
        save_format: Format to use for saving sample images
            (if `save_to_dir` is set).
    """

    def __init__(self, directory, image_data_generator,
                 target_size=(256, 256), color_mode='rgb',
                 classes=None, class_mode='categorical',
                 batch_size=32, shuffle=True, seed=None,
                 data_format=None,
                 save_to_dir=None, save_prefix='', save_format='png',
                 follow_links=False, subset=None, interpolation='nearest'):
        if data_format is None:
            data_format = K.image_data_format()
        self.directory = directory
        self.image_data_generator = image_data_generator
        self.target_size = tuple(target_size)
        if color_mode not in {'rgb', 'grayscale'}:
            raise ValueError('Invalid color mode:', color_mode,
                             '; expected "rgb" or "grayscale".')
        self.color_mode = color_mode
        self.data_format = data_format
        if self.color_mode == 'rgb':
            if self.data_format == 'channels_last':
                self.image_shape = self.target_size + (3,)
            else:
                self.image_shape = (3,) + self.target_size
        else:
            if self.data_format == 'channels_last':
                self.image_shape = self.target_size + (1,)
            else:
                self.image_shape = (1,) + self.target_size
        self.classes = classes
        if class_mode not in {'categorical', 'binary', 'sparse',
                              'input', None}:
            raise ValueError('Invalid class_mode:', class_mode,
                             '; expected one of "categorical", '
                             '"binary", "sparse", "input"'
                             ' or None.')
        self.class_mode = class_mode
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format
        self.interpolation = interpolation

        white_list_formats = {'png', 'jpg', 'jpeg', 'bmp'}

        # first, count the number of samples and classes
        self.samples = 0

        if not classes:
            classes = []
            for subdir in sorted(os.listdir(directory)):
                if os.path.isdir(os.path.join(directory, subdir)):
                    classes.append(subdir)
        self.num_class = len(classes)
        self.class_indices = dict(zip(classes, range(len(classes))))

        def _recursive_list(subpath):
            return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])

        pool = multiprocessing.pool.ThreadPool()
        function_partial = partial(_count_valid_files_in_directory,
                                   white_list_formats=white_list_formats,
                                   split=None,
                                   follow_links=follow_links)
        self.samples = sum(pool.map(function_partial,
                                    (os.path.join(directory, subdir)
                                     for subdir in classes)))

        print('Found %d images belonging to %d classes.' % (self.samples, self.num_class))

        # second, build an index of the images in the different class subfolders
        results = []

        self.filenames = []
        self.classes = np.zeros((self.samples,), dtype='int32')
        i = 0
        for dirpath in (os.path.join(directory, subdir) for subdir in classes):
            results.append(pool.apply_async(_list_valid_filenames_in_directory,
                                            (dirpath, white_list_formats, None,
                                             self.class_indices, follow_links)))
        for res in results:
            classes, filenames = res.get()
            self.classes[i:i + len(classes)] = classes
            self.filenames += filenames
            i += len(classes)
        pool.close()
        pool.join()
        super(DirectoryIteratorWithFname, self).__init__(self.samples, batch_size, shuffle, seed)

    def next(self):
        """For python 2.x.
        # Returns
            The next batch.
        """
        with self.lock:
            index_array = next(self.index_generator)
            current_batch_size = index_array.size

        # The transformation of images is not under thread lock
        # so it can be done in parallel
        batch_x = np.zeros((current_batch_size,) + self.image_shape, dtype=K.floatx())
        batch_fn = ['' for _ in range(current_batch_size)]
        grayscale = self.color_mode == 'grayscale'
        # build batch of image data
        for i, j in enumerate(index_array):
            fname = self.filenames[j]
            img = load_img(os.path.join(self.directory, fname),
                           grayscale=grayscale,
                           target_size=self.target_size,
                           interpolation=self.interpolation)
            x = img_to_array(img, data_format=self.data_format)
            x = self.image_data_generator.random_transform(x)
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
            batch_fn[i] = fname.split(os.path.sep)[-1]
        # optionally save augmented images to disk for debugging purposes
        if self.save_to_dir:
            for i, j in enumerate(index_array):
                img = array_to_img(batch_x[i], self.data_format, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=j,
                                                                  hash=np.random.randint(1e7),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        # build batch of labels
        if self.class_mode == 'input':
            batch_y = batch_x.copy()
        elif self.class_mode == 'sparse':
            batch_y = self.classes[index_array]
        elif self.class_mode == 'binary':
            batch_y = self.classes[index_array].astype(K.floatx())
        elif self.class_mode == 'categorical':
            batch_y = np.zeros((len(batch_x), self.num_class), dtype=K.floatx())
            for i, label in enumerate(self.classes[index_array]):
                batch_y[i, label] = 1.
        else:
            return batch_x, batch_fn
        return batch_x, batch_y, batch_fn


#!/usr/bin/env python3
import sys

def substrs(haystack, needle):
    matches = []

    offset = 0
    while True:
        match = haystack.find(needle, offset)

        if match == -1:
            break

        matches.append(match)
        offset = match + 1

    return tuple(n+1 for n in matches)

if __name__ == '__main__':
    try:
        filename = sys.argv[1]
    except IndexError:
        print('Missing filename', file=sys.stderr)
        sys.exit(1)

    with open(filename) as f:
        s = f.readline().strip()
        t = f.readline().strip()

    print(' '.join(str(n) for n in substrs(s, t)))


"""
Script goal, to produce trends in netcdf files
This script can also be used in P03 if required

"""
#==============================================================================

__title__ = "Global Vegetation Trends"
__author__ = "Arden Burrell"
__version__ = "v1.0(28.03.2019)"
__email__ = "arden.burrell@gmail.com"

#==============================================================================
# +++++ Check the paths and set ex path to fireflies folder +++++
import os
import sys
if not os.getcwd().endswith("fireflies"):
	if "fireflies" in os.getcwd():
		p1, p2, _ =  os.getcwd().partition("fireflies")
		os.chdir(p1+p2)
	else:
		raise OSError(
			"This script was called from an unknown path. CWD can not be set"
			)
sys.path.append(os.getcwd())

#==============================================================================
# Import packages
import numpy as np
import pandas as pd
import argparse
import datetime as dt
from collections import OrderedDict
import warnings as warn
from netCDF4 import Dataset, num2date, date2num
from scipy import stats
import xarray as xr
from dask.diagnostics import ProgressBar
from numba import jit
import bottleneck as bn
import scipy as sp
import glob
from scipy import stats
import statsmodels.stats.multitest as smsM

# Import plotting and colorpackages
import matplotlib.pyplot as plt
import matplotlib.colors as mpc
import matplotlib as mpl
import palettable
import seaborn as sns
import cartopy.crs as ccrs
import cartopy.feature as cpf
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
# Import debugging packages
import ipdb

print("numpy version  : ", np.__version__)
print("pandas version : ", pd.__version__)
print("xarray version : ", xr.__version__)
#==============================================================================
def main():
	# =========== Create the summary of the datasets to be analyised ==========
	data= OrderedDict()

	data["Terrraclim_ppt"] = ({
		'fname':"./data/cli/1.TERRACLIMATE/TerraClimate_stacked_ppt_1977to2017_ppt_yearsum_20window.nc",
		'var':"ppt", "gridres":"terraclimate", "region":"Global", "Periods":["AnnualMax"]
		})
	data["Terrraclim_tmean"] = ({
		'fname':"./data/cli/1.TERRACLIMATE/TerraClimate_stacked_tmean_1977to2017_annualmean_20yearwindow.nc",
		'var':"tmean", "gridres":"terraclimate", "region":"Global", "Periods":["AnnualMax"]
		})

	fname = "./data/cli/1.TERRACLIMATE/TerraClimate_stacked_ppt_1977to2017_ppt_yearsum_20window_trend2.nc"
	Plot_Trend(xr.open_dataset(fname), ["slope"], "Terrraclim_ppt", "ppt", "polyfit", "Terraclimate", "Global")
	fname = "./data/cli/1.TERRACLIMATE/TerraClimate_stacked_tmean_1977to2017_annualmean_20yearwindow_slope.nc"
	Plot_Trend(xr.open_dataset(fname), ["slope"], "Terrraclim_tmean", "tmean", "polyfit", "Terraclimate", "Global")
	for method in ["polyfit", "scipyols", "theilsen"]:
		for dt in data:
			print (dt, method)
			# ========== FIt a theilsen slope estimation ==========
			trendmapper(dt,
					data[dt]["fname"], data[dt]["var"], method,
					data[dt]["gridres"], data[dt]["region"])

def trendmapper(
	dataset, fname, var, method, gridres, region, fdpath="", force = False, plot=True):
	"""
	Master function for trend finder
	args:
		dataset: str
			the code for the dataset
		fname: str
			filename of the netcdf file to be opened
		var: str
			name of the variable in the netcdf

	"""

	# ========== open the dataset and pull the values
	if type(fname) == str:
		ds       = xr.open_dataset(fname)
		global_attrs = GlobalAttributes(ds, var)
	else:
		if dataset == "MODISaqua":
			fouts = "./data/veg/MODIS/aqua/%s_AnnualMax.nc" % dataset
		# files to be removed
		cleanup = []
		if not os.path.isfile(fouts):
			for fls in fname:
				# ========== open the file for a given year ==========
				fileouts = fls[:-3]+"_AnnualMax.nc"
				cleanup.append(fileouts)

				if not os.path.isfile(fileouts):
					dsmf = xr.open_dataset(fls, chunks={"latitude":480})
					global_attrs = GlobalAttributes(dsmf, var)

					# ========== get the max value for the year ==========

					# ipdb.set_trace()
					dsp   = dsmf.groupby("time.year").max(dim="time")
					tm = [dt.datetime(int(year) , 6, 30) for year in dsp.year]
					dsp = dsp.rename({"year":"time"})
					dsp["time"] = pd.to_datetime(tm)
					dsp.attrs = global_attrs

					# ipdb.set_trace()

					print("starting write")
					encoding = OrderedDict()
					# ipdb.set_trace()
					# sys.exit()
					encoding[var] = ({'shuffle':True,
						# 'chunksizes':[1, dsp.latitude.shape[0], dsp.longitude.shape[0]],
						'zlib':True,
						'complevel':6})

					delayed_obj = dsp.to_netcdf(fileouts,
						format         = 'NETCDF4',
						encoding       = encoding,
						unlimited_dims = ["time"],
						compute=False)
					with ProgressBar():
						results = delayed_obj.compute()
					dsp.close()
					dsmf.close()
			# ========== Make a joined file name ==========
			jfname = " ".join(cleanup)
			# print mergetime
			subp.call(
				"cdo -P 4 -b F64 mergetime %s %s" % (jfname, fouts),
				shell=True
				)
			# Open the saved dataset
			ds = xr.open_dataset(fouts)
			# remove the interum files
			for fles in cleanup:
				os.remove(fles)

		else:ds = xr.open_dataset(fouts)

		# warn.warn(" i need to save it out and reload everything")
		# ipdb.set_trace()

	yr_start = pd.to_datetime(ds.time.min().values).year
	endyr   = pd.to_datetime(ds.time.max().values).year
	# ipdb.set_trace()

	# ========== Create the outfile name ==========
	fout = './results/netcdf/%s_%s_%s_%sto%d_%s%s.nc' % (
		dataset,  var, method, yr_start, endyr,region, gridres)

	# ========== Check if the file already exists ==========
	if all([os.path.isfile(fout), not force]):
		warn.warn("Loading existing file, force is needed to overwrite")
		ds_trend = xr.open_dataset(fout)
		kys = [n for n in ds_trend.data_vars]
	else:
		results     = []

		# ========== Create the global attributes ==========
		dst = ds[var]
		if dataset == "GIMMS31v10":
			dst /= 10000.0

		# ========== Calculate the trend ==========
		if (dst.nbytes * 1e-9) <  16:
			trends, kys = _fitvals(dst, method=method)
		else:
			trends, kys = _multifitvals(dst, method=method)
		# Correct for multiple comparisons
		if "pvalue" in kys:
			trends, kys = MultipleComparisons(trends, kys, aplha = 0.10)

		results.append(trends)

		layers, encoding = dsmaker(ds, var, results, kys, method)
		ds_trend = xr.Dataset(layers, attrs= global_attrs)

		try:
			print("Starting write of data")
			ds_trend.to_netcdf(fout,
				format         = 'NETCDF4',
				encoding       = encoding,
				unlimited_dims = ["time"])
		except Exception as e:
			print(e)
			warn.warn(" \n something went wrong with the save, going interactive")
			ipdb.set_trace()

	if plot:
		Plot_Trend(ds_trend, kys, dataset, var, method, gridres, region)

		# get the value
#==============================================================================

def Plot_Trend(ds_trend, kys, dataset, var, method, gridres, region):
	"""
	Function to build global trend maps
	"""
	# ========== Build all the plots ==========
	# +++++ Plot number +++++
	pn = 1

	# ========== create the colormap ==========
	cmap, vmin, vmax = cbvals(var, "slope")
	# plt.figure(1, dpi=600)
	ax = plt.subplot(projection=ccrs.PlateCarree())
	ax.add_feature(cpf.OCEAN, facecolor="w", alpha=1, zorder=100)
	ax.add_feature(cpf.COASTLINE, zorder=101)
	ax.add_feature(cpf.BORDERS, linestyle='--', zorder=102)
	ax.add_feature(cpf.LAKES, alpha=0.5, zorder=103)
	ax.add_feature(cpf.RIVERS, zorder=104)
	# add lat long linse
	gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
		linewidth=1, color='gray', alpha=0.5, linestyle='--')
	gl.xlabels_top = False
	gl.ylabels_right = False

	gl.xformatter = LONGITUDE_FORMATTER
	gl.yformatter = LATITUDE_FORMATTER

	ds_trend.slope.plot(
		transform=ccrs.PlateCarree(), ax=ax,
		cmap=cmap, vmin=vmin, vmax=vmax)
	fig = plt.gcf()
	fig.set_size_inches(41, 20)
	plt.savefig("./plots/Meeting/%s_%s_%s_slope_%s.png" % (dataset, method, var, region))#, dp1=400)
	# plt.savefig("./plots/Meeting/%s_%s_%s_slope_%s.pdf" % (dataset, method, var, region))#, dp1=400)
	plt.show()
	# plt.show()
	# plt.coloes

	ax = plt.subplot(projection=ccrs.PlateCarree())
	ax.add_feature(cpf.OCEAN, facecolor="w", alpha=1, zorder=100)
	ax.add_feature(cpf.COASTLINE, zorder=101)
	ax.add_feature(cpf.BORDERS, linestyle='--', zorder=102)
	ax.add_feature(cpf.LAKES, alpha=0.5, zorder=103)
	ax.add_feature(cpf.RIVERS, zorder=104)
	# add lat long linse
	gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
		linewidth=1, color='gray', alpha=0.5, linestyle='--')
	gl.xlabels_top = False
	gl.ylabels_right = False

	gl.xformatter = LONGITUDE_FORMATTER
	gl.yformatter = LATITUDE_FORMATTER
	try:

		ds_trend.slope.sel({'latitude':slice(56, 49), "longitude":slice(103, 123)}).plot(
			transform=ccrs.PlateCarree(), ax=ax,
			cmap=cmap, vmin=vmin, vmax=vmax)
	except ValueError:
		ds_trend.slope.sel({'lat':slice(56, 49), "lon":slice(103, 123)}).plot(
			transform=ccrs.PlateCarree(), ax=ax,
			cmap=cmap, vmin=vmin, vmax=vmax)
	plt.savefig("./plots/Meeting/%s_%s_%s_slope_StudyArea.png" % (dataset, method, var))#, dp1=400)
	plt.savefig("./plots/Meeting/%s_%s_%s_slope_StudyArea.pdf" % (dataset, method, var))#, dp1=400)
	plt.show()

	ipdb.set_trace()

def cbvals(var, ky):

	"""Function to store all the colorbar infomation i need """
	cmap = None
	vmin = None
	vmax = None
	if ky == "slope":
		if var == "tmean":
			vmax =  0.07
			vmin = -0.07
			cmap = mpc.ListedColormap(palettable.cmocean.diverging.Balance_20.mpl_colors)
		elif var =="ppt":
			vmin = -5.0
			vmax =  5.0
			cmap = mpc.ListedColormap(palettable.cmocean.diverging.Curl_20_r.mpl_colors)
	elif ky == "pvalue":
		cmap = mpc.ListedColormap(palettable.matplotlib.Inferno_20.hex_colors)
		vmin = 0.0
		vmax = 1.0
	elif ky == "rsquared":
		cmap = mpc.ListedColormap(palettable.matplotlib.Viridis_20.hex_colors)
		vmin = 0.0
		vmax = 1.0
		# cmap =
	elif ky == "intercept":
		cmap = mpc.ListedColormap(palettable.cmocean.sequential.Ice_20_r.mpl_colors)
		if var == "tmean":
			# vmax =  0.07
			# vmin = -0.07
			# cmap = mpc.ListedColormap(palettable.cmocean.diverging.Balance_20.mpl_colors)
			# ipdb.set_trace()
			pass
		elif var =="ppt":
			vmin = 0
			vmax = 1000
			# cmap = mpc.ListedColormap(palettable.cmocean.diverging.Curl_20_r.mpl_colors)

	return cmap, vmin, vmax

def GlobalAttributes(ds, var):
	"""
	Creates the global attributes for the netcdf file that is being written
	these attributes come from :
	https://www.unidata.ucar.edu/software/thredds/current/netcdf-java/metadata/DataDiscoveryAttConvention.html
	args
		ds: xarray ds
			Dataset containing the infomation im intepereting
		var: str
			name of the variable
	returns:
		attributes 	Ordered Dictionary cantaining the attribute infomation
	"""
	# ========== Create the ordered dictionary ==========
	attr = OrderedDict()

	# fetch the references for my publications
	# pubs = puplications()

	# ========== Fill the Dictionary ==========

	# ++++++++++ Highly recomended ++++++++++
	attr["title"]               = "Trend in Climate (%s)" % (var)
	attr["summary"]             = "Annual and season trends in %s" % var
	attr["Conventions"]         = "CF-1.7"

	# ++++++++++ Data Provinance ++++++++++
	attr["history"]             = "%s: Netcdf file created using %s (%s):%s by %s" % (
		str(pd.Timestamp.now()), __title__, __file__, __version__, __author__)
	attr["history"]            += ds.history

	attr["creator_name"]        = __author__
	attr["creator_url"]         = "ardenburrell.com"
	attr["creator_email"]       = __email__
	attr["institution"]         = "University of Leicester"
	attr["date_created"]        = str(pd.Timestamp.now())

	# ++++++++++ Netcdf Summary infomation ++++++++++
	attr["time_coverage_start"] = str(dt.datetime(ds['time.year'].min(), 1, 1))
	attr["time_coverage_end"]   = str(dt.datetime(ds['time.year'].max() , 12, 31))

	return attr

def dsmaker(ds, var, results, keys, method):
	"""
	Build a summary of relevant paramters
	args
		ds: xarray ds
			Dataset containing the infomation im intepereting
		var: str
			name of the variable
	return
		ds 	xarray dataset
	"""
	# sys.exit()
	tm = [dt.datetime(ds['time.year'].max() , 12, 31)]
	times = OrderedDict()
	# tm    = [dt.datetime(yr , 12, 31) for yr in start_years]
	# tm    = [date]
	times["time"] = pd.to_datetime(tm)

	times["calendar"] = 'standard'
	times["units"]    = 'days since 1900-01-01 00:00'

	times["CFTime"]   = date2num(
		tm, calendar=times["calendar"], units=times["units"])

	dates = times["CFTime"]

	try:
		lat = ds.lat.values
		lon = ds.lon.values
	except AttributeError:
		lat = ds.latitude.values
		lon = ds.longitude.values
	# dates = [dt.datetime(yr , 12, 31) for yr in start_years]
	# ipdb.set_trace()
	# ========== Start making the netcdf ==========
	layers   = OrderedDict()
	encoding = OrderedDict()
	# ========== loop over the keys ==========
	for pos in range(0, len(keys)):
		try:
			# ipdb.set_trace()
			if type(results[0]) == np.ndarray:
				Val = results[pos][np.newaxis,:, :]
			else:
				# multiple variables
				Val = np.stack([res[pos] for res in results])
			ky = keys[pos]

			# build xarray dataset
			DA=xr.DataArray(Val,
				dims = ['time', 'latitude', 'longitude'],
				coords = {'time': dates,'latitude': lat, 'longitude': lon},
				attrs = ({
					'_FillValue':9.96921e+36,
					'units'     :"1",
					'standard_name':ky,
					'long_name':"%s %s" % (method, ky)
					}),
			)

			DA.longitude.attrs['units'] = 'degrees_east'
			DA.latitude.attrs['units']  = 'degrees_north'
			DA.time.attrs["calendar"]   = times["calendar"]
			DA.time.attrs["units"]      = times["units"]
			layers[ky] = DA
			encoding[ky] = ({'shuffle':True,
				# 'chunksizes':[1, ensinfo.lats.shape[0], 100],
				'zlib':True,
				'complevel':5})

		except Exception as e:
			warn.warn("Code failed with: \n %s \n Going Interactive" % e)
			ipdb.set_trace()
			raise e
	return layers, encoding

def MultipleComparisons(trends, kys, aplha = 0.10, MCmethod="fdr_by"):
	"""
	Takes the results of an existing trend detection aproach and modifies them to
	account for multiple comparisons.
	args
		trends: list
			list of numpy arrays containing results of trend analysis
		kys: list
			list of what is in results
		years:
			years of accumulation

	"""
	if MCmethod == "fdr_by":
		print("Adjusting for multiple comparisons using Benjamini/Yekutieli")
	elif MCmethod == "fdr_bh":
		print("Adjusting for multiple comparisons using Benjamini/Hochberg")
	else:
		warn.warn("unknown MultipleComparisons method, Going Interactive")
		ipdb.set_trace()

	# ========== Locate the p values and reshape them into a 1d array ==========
	# ++++++++++ Find the pvalues ++++++++++
	index      = kys.index("pvalue")
	pvalue     = trends[index]
	isnan      = np.isnan(pvalue)

	# ++++++++++ pull out the non nan pvalus ++++++++++
	# pvalue1d = pvalue.flatten()
	pvalue1d   = pvalue[~isnan]
	# isnan1d  = isnan.flatten()

	# =========== Perform the MC correction ===========
	pvalue_adj =  smsM.multipletests(pvalue1d, method=MCmethod, alpha=0.10)

	# ++++++++++ reformat the data into array ++++++++++
	MCR =  ["Significant", "pvalue_adj"]
	for nm in MCR:
		# make an empty array
		re    = np.zeros(pvalue.shape)
		re[:] = np.NAN
		if nm == "Significant":
			re[~isnan] = pvalue_adj[MCR.index(nm)].astype(int).astype(float)
		else:
			re[~isnan] = pvalue_adj[MCR.index(nm)]

		# +++++ add the significant and adjusted pvalues to trends+++++
		trends.append(re)
		kys.append(nm)
	return trends, kys

def _fitvals(dvt, method="polyfit"):
	"""
	Takes the ds[var] and performs some form of regression on it
	args
		dvt: xarray data array
			the values to be regressed
		method: str
			the regression approach to take
	"""
	# ========== Get the values ==========
	vals  = dvt.values
	# except MemoryError:
	# 	ipdb.set_trace()

	# ========== Convert the time into years ==========
	try:
		years = pd.to_datetime(dvt.time.values).year
		t0 = pd.Timestamp.now()
		print("testing with %s from %d to %d starting at: %s" % (
			method, pd.to_datetime(dvt.time.values).year.min(),
			pd.to_datetime(dvt.time.values).year.max(), str(t0)))

	except AttributeError:
		years = pd.to_datetime(dvt.year.values).year
		t0 = pd.Timestamp.now()
		print("testing with %s from %d to %d starting at: %s" % (
			method, pd.to_datetime(dvt.year.values).year.min(),
			pd.to_datetime(dvt.year.values).year.max(), str(t0)))

	# ========== Reshape the datainto two dims  ==========
	vals2 = vals.reshape(len(years), -1)

	# ========== pass the results to the specific regression function  ==========
	if method=="polyfit":
		# Do a first-degree polyfit
		vals2[np.isnan(vals2)] = 0
		regressions = np.polyfit(years, vals2, 1)
		ipdb.set_trace()
		regressions[regressions== 0] = np.NAN
		trends = [regressions[0,:].reshape(vals.shape[1], vals.shape[2])]
		kys = ["slope"]
		ipdb.set_trace()

	elif method == "theilsen":
		regressions = alongaxFAST(vals2, scipyTheilSen)
		trds = regressions.reshape(4, vals.shape[1], vals.shape[2])
		trends = []
		for n in range(0, trds.shape[0]):
			trends.append(trds[n, :, :])
		kys = ["slope", "intercept", "rho", "pvalue"]

	elif method == "scipyols":
		# regressions = alongax(vals2, scipyols)
		regressions = alongaxFAST(vals2, scipyols)
		trds = regressions.reshape(4, vals.shape[1], vals.shape[2])
		trends = []
		for n in range(0, trds.shape[0]):
			trends.append(trds[n, :, :])

		kys = ["slope", "intercept", "rsquared", "pvalue"]

	tdelta = pd.Timestamp.now() - t0
	print("\n Time taken to get regression coefficients using %s: %s" % (method, str(tdelta)))
	# ipdb.set_trace()
	return trends, kys

def alongaxFAST(array, myfunc, lineflick=10000):
	""" Fastest wave i've yet found to loop over an entire netcdf file
	array 2d numpy array
	myfunc function i want to apply
	lineflick frequency that i want to see the lines, increasing this number
		increases speed
	returns
		res 2d array with the results
	"""
	# build an empyt array to hold the result
	# res = np.zeros((array.shape[1], 4))
	res = np.zeros((4, array.shape[1]))
	res[:] = np.NAN

	# locate and remove any nan rows
	ana = ~bn.anynan(array, axis=0)
	array2 = array[:, ana]

	# ========== build a holder ==========
	vals = np.zeros((4, array2.shape[1]))

	# ========== get the starting time ==========
	t0 = pd.Timestamp.now()

	for line in range(0, array2.shape[1]):
		if (line % lineflick == 0):
			string = ("\rRegression climate: line: %d of %d" %
						(line, array2.shape[1]))
			if line > 0:
				# TIME PER LINEFLICK
				lfx = (pd.Timestamp.now()-t0)/line
				lft = str((lfx*lineflick))
				trm = str(((array2.shape[1]-line)*(lfx)))

				string += (" t/%d lines: %s. ~eta: %s" % (
					lineflick,lft, trm) )

			sys.stdout.write(string)
			sys.stdout.flush()

		out = myfunc(array2[:, line])
		# vals.append(out)
		vals[:, line] = out
	res[:, ana] = vals
	return res

def scipyTheilSen(array):
	"""
	Function for rapid TheilSen slop estimation with time.
	the regression is done with  an independent variable
	rangeing from 0 to array.shape to make the intercept
	the start which simplifies calculation

	args:
		array 		np : numpy array of annual max VI over time
	return
		result 		np : slope, intercept
	"""
	try:
		# if bn.allnan(array):
		# 	return np.array([np.NAN, np.NAN, np.NAN, np.NAN])

		slope, intercept, _, _ = stats.mstats.theilslopes(
			array, np.arange(array.shape[0]))
		rho, pval = stats.spearmanr(
			array, np.arange(array.shape[0]))
		# change = (slope*array.shape[0])
		return np.array([slope, intercept, rho, pval])

	except Exception as e:
		print(e)
		warn.warn("unhandeled Error has occured")
		ipdb.set_trace()
		return np.array([np.NAN, np.NAN, np.NAN, np.NAN])

# @jit
def scipyols(array):
	"""
	Function for rapid OLS with time. the regression is done with
	an independent variable rangeing from 0 to array.shape to make
	the intercept the start which simplifies calculation
	args:
		array 		np : numpy array of annual max VI over time
	return
		result 		np : change(total change between start and end)
						 slopem intercept, rsquared, pvalue, std_error
	"""
	# +++++ Get the OLS +++++
	try:
		# if bn.allnan(array):
			# return np.array([np.NAN, np.NAN, np.NAN, np.NAN])

		slope, intercept, r_value, p_value, std_err = stats.linregress(np.arange(array.shape[0]), array)
		# +++++ calculate the total change +++++
		# change = (slope*array.shape[0])
		# +++++ return the results +++++
		return np.array([slope, intercept, r_value**2, p_value])
	except Exception as e:
		# print(e)
		# warn.warn("unhandeled Error has occured")
		# ipdb.set_trace()
		return np.array([np.NAN, np.NAN, np.NAN, np.NAN])

if __name__ == '__main__':
	main()


#-*- coding:utf-8 -*-
#import csv
import os, shutil
import time
#import threading

#from kivy.uix.label import Label
#from kivy.uix.button import Button

#import pyaudio
#import wave
import playsound as pls

from gtts import gTTS
import speech_recognition as sr

import songBox_playerPyaudio as sbp

nowDate =  time.strftime("%Y_%m_%d")

baseDir = os.getcwd()#os.path.abspath('py_song')#workging 폴더
mp3Dir = os.path.join(baseDir, "songBox_list")#노래 폴더
fontDir = os.path.join(baseDir, "songBox_font")#글자 폴더
imgDir = os.path.join(baseDir, "songBox_img")#그림 폴더
dataDir = os.path.join(mp3Dir, "0_file_data")#데이터목록 csv 저장폴더
userListDir = os.path.join(mp3Dir, "1_dir_userlist")#userlist 폴더
singerListDir = os.path.join(mp3Dir, "2_dir_singerlist")#userlist 폴더
ttsDir = os.path.join(mp3Dir, "3_tts")#tts 폴더
ignoreFile = ['ffmpeg','youtube-dl.exe','.DS_Store','list_data','0_file_data','1_dir_userlist','2_dir_singerlist','3_tts']#dir 안에 존재해야만하는 파일 but mp3 아닌 파일 목록

winColor =[211/255, 211/255, 211/255, 1]#[189/255,183/255,110/255,1]#[211/255,211/255,211/255,1]#
boxColor = [188/255, 143/255, 143/255, 1]#[110/255,0/255,0/255,1]#[105/255,105/255,105/255,1]#
textColor = [255/255, 255/255, 255/255, 1]
stopColor = [105/255,105/255,105/255,1]#gold[189/255,183/255,110/255,1]#red[110/255,0/255,0/255,1]

bartextColor =[189/255,183/255,110/255,1]#deepblue+green[30/255,160/255,180/255,1]#grey[105/255,105/255,105/255,1]#red[110/255,0/255,0/255,1]

soundbar_fontsize="22sp"
menu_fontsize="20sp"
text_fontsize="17sp"

#tempObj = Label(text='1')
#tempLabel=Label(text='ready')
#tempBtn=Button(text='temp')

#==============first msg========================================================
def play_msg(msg):
    print(f"play_msg:{play_msg}")
    get_msg = msg
    flag = False

    if msg == 'Tell me the song title':
        pls.playsound(f'{ttsDir}\\startmsg.mp3', False)
        duration = sbp.cal_duration(f"{ttsDir}\\startmsg.mp3")
        time.sleep(duration)
    else:
        tts = gTTS(text=f"{get_msg}",lang="en",tld="com")
        tts.save(f"{ttsDir}\\msg.mp3")

        pls.playsound(f'{ttsDir}\\msg.mp3', False)
        duration = sbp.cal_duration(f"{ttsDir}\\msg.mp3")
        time.sleep(duration)
        os.remove(f"{ttsDir}\\msg.mp3")

    if msg != 'Tell me the song title':
        flag = True

    return flag

#==============yes or no========================================================
def yesno_voice():
    print(f"start_yesno_voice")
    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()
    audio=False

    while audio == False:
        try:
            with m as source:
                print(f'yes or no:')
                audio = r.listen(source)

        except Exception as errmsg:
                yesno_msg = f"{errmsg}retry."
                print(f"{errmsg}retry.")

                return yesno_msg

    try:
        yesno_msg=r.recognize_google(audio)
        print(f'answer_yes or no:{yesno_msg}')

        return yesno_msg.lower()

    except Exception as errmsg:
        yesno_msg=errmsg
        return yesno_msg

#==============start voice conversation=========================================
def searchsong_voice():
    print(f"start_searchsong_voice")
    msg = 'Tell me the song title'
    yesno_msg = ""
    yesno_msg_sec = ""
    flag = play_msg(msg)
    flag = False
    audio = False

    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()

    #==============소리 감지할 때까지 ===============================================
    while audio == False:

        try:
            with m as source:
                print(f'speak:')
                audio = r.listen(source)
                flag=True

            #==============time check===========================================
            if time.time()-start > 10:
                print(f"I waited {time.time()-start} seconds")
                msg = f"I waited {int(time.time()-start)} seconds. Will I wait for you to speak?"
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"wait_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"wait_yes{yesno_msg}")
                        return searchsong_voice()

                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

        #==============exception check==========================================
        except Exception as errmsg:
                print(f"{errmsg}")
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"first_except_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"first_except_yes{yesno_msg}")
                        return searchsong_voice()
                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

    #==============소리 감지 & flag == True=======================================
    if flag == True and audio != False:
        try:
            speechSong=r.recognize_google(audio)
            print(f'You said:{speechSong}')
            msg = f'You said:{speechSong}. Did you say this song title?'
            play_msg(msg)

            while yesno_msg !="no" and yesno_msg !="yes":

                yesno_msg = yesno_voice()

                if yesno_msg == "no":
                    msg = f'Would you like to try again?'
                    print(f"second_no{yesno_msg}")
                    play_msg(msg)
                    yesno_msg_sec

                elif yesno_msg == "yes":
                    print(f"second_yes{yesno_msg}")
                    return search_song(speechSong)

                else:
                    msg = f"I can't understand your answer. Tell me one more time."
                    play_msg(msg)

            if yesno_msg == "no":
                while yesno_msg_sec !="no" or yesno_msg_sec !="yes":
                    yesno_msg_sec = yesno_voice()

                    if yesno_msg_sec == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"second_no_no{yesno_msg_sec}")
                        return False, msg

                    elif yesno_msg_sec == "yes":
                        print(f"second_no_yes{yesno_msg_sec}")
                        return searchsong_voice()
                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

        except Exception as errmsg:
                print(f"I can't recognize the voice.{errmsg}")
                msg = f"I can't recognize the voice. Would you like to try again?"
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"second_except_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"second_except_yes{yesno_msg}")
                        return searchsong_voice()

                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

#==============search song > return result =====================================
def search_song(speechSong):
    global GET_PLAY_BTN

    yesno_msg=''
    print(f'start search_song:{speechSong}')

    try:
        songList=os.listdir(f'{mp3Dir}')

        for i in songList:
            if i not in ignoreFile:

                title=i.split('_')

                if speechSong.lower() == title[0].lower():
                    print(f"find {i}")

                    return True, i

        msg = f"This song title {speechSong} could not be found. Would you like to try again?"
        res = play_msg(msg)

        while yesno_msg !="no" and yesno_msg !="yes":
            yesno_msg = yesno_voice()

            if yesno_msg == "no":
                msg = f"Ok. Bye"
                flag = play_msg(msg)
                print(f'search_song__no:{yesno_msg}')
                return False, msg

            elif yesno_msg == "yes":
                print(f'search_song__yes:{yesno_msg}')
                return searchsong_voice()

            else:
                msg = f"I can't understand your answer. Tell me one more time."
                flag = play_msg(msg)

    except Exception as errmsg:
            print(f"I can't find this song {speechSong}. {errmsg}")
            msg = f"I can't find this song {speechSong}. Would you like to try again?"
            flag = play_msg(msg)

            while yesno_msg !="no" and yesno_msg !="yes":
                yesno_msg = yesno_voice()

                if yesno_msg == "no":
                    msg = f"Ok. Bye"
                    flag = play_msg(msg)
                    print(f'search_song_except_no:{yesno_msg}')
                    return False, msg

                elif yesno_msg == "yes":
                    print(f'search_song_except_yes:{yesno_msg}')
                    return searchsong_voice()

                else:
                    msg = f"I can't understand your answer. Tell me one more time."
                    flag = play_msg(msg)

"""
def recognize_test(play_btn):
    #print(play_btn,pause_btn)
    get_PLAY_BTN = play_btn
    #get_PAUSE_BTN = pause_btn
    #os.system(f"open {baseDir}/searhsong.mp3")
    #searchsong = SoundLoader.load(f"{baseDir}/searchsong.mp3")
    #searchsong.play()
    #searchsong.volume = 1.0
    #searchsong.loop = False
    audio = False
    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()

    while audio == False:
        try:
            with m as source:
                print(f'speak:')
                audio = r.listen(source)
            if time.time()-start > 10:
                sbp.get_restart()
                print("Waited 10 seconds")
                return 0
        except Exception as msg:
                print(f"{msg}retry.")
                return 0

    try:
        speechText=r.recognize_google(audio)
        print(f'You said:{speechText}')

    except Exception as msg:
            sbp.get_restart()
            print(f"can't recognize the voice. {msg}")
            return 0

    if audio != False:

        songList=os.listdir(f'{mp3Dir}')
        for i in songList:
            if i not in ignoreFile:
                title=i.split('_')
                if speechText.lower() == title[0].lower():
                    print(f"find {i}")

                    #get_PAUSE_BTN.text = "|||"
                    #get_PAUSE_BTN.color = boxColor
                    playTitle = [i]
                    playing = [tempObj,]
                    replay=True
                    playnum=0

                    sbp.get_restart()
                    thread_one = threading.Thread(target = sbp.get_playThread, args = (playTitle,replay,playing,get_PLAY_BTN,tempBtn,tempLabel,playnum), daemon=True)
                    thread_one.start()
                    return 0

        #nosong = SoundLoader.load(f"{baseDir}/nosong.mp3")
        #nosong.play()
        #nosong.loop = False
        #os.system(f"open {baseDir}/nosong.mp3")
        sbp.get_restart()
        print(f"can't find {speechText}")
        return 0
"""


# coding=utf-8
# Copyright 2018 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Download manager interface.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import contextlib
import os

import concurrent.futures
import six
from tensorflow import gfile

from tensorflow_datasets.core import utils
from tensorflow_datasets.core.download import local_backend
from tensorflow_datasets.core.download import util
from tensorflow_datasets.core.proto import download_generated_pb2 as download_pb2

# Number of thread to use to parallelize the extractions
_NUM_PARALLEL_DOWNLOADS = 50
_NUM_PARALLEL_EXTRACTS = 1

TAR_EXT = ['.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tbz', '.tb2']
# Tuple to match extensions with extraction function.
ExtractFormat = collections.namedtuple('ExtractFormat', 'ext, fn')

class DownloadManager(object):
  """Class which manages the download and extraction of data.

  This has the following advantage:
   * Remove the boilerplate code common to each dataset as now the user only
     has to provide urls.
   * Allow multiple download backends (local and cloud). Otherwise the user
     may be tempted to use standard Python function (ex: tarfile) which may
     not be compatible with one of the backend.
   * Allow to launch multiple download in parallel.
   * Better ressources management (ex: a same url used by multiple
     datasets is downloaded/extracted only once).

  The function members accept either plain value, or values wrapped into list
  or dict. Giving a data structure will parallelize the downloads.

  Example:

    # Sequential download: str -> str
    train_dir = dl_manager.download_and_extract('https://abc.org/train.tar.gz')
    test_dir = dl_manager.download_and_extract('https://abc.org/test.tar.gz')

    # Parallel download: list -> list
    image_files = dl_manager.download(
        ['https://a.org/1.jpg', 'https://a.org/2.jpg', ...])

    # Parallel download: dict -> dict
    data_dirs = dl_manager.download_and_extract({
       'train': 'https://abc.org/train.zip',
       'test': 'https://abc.org/test.zip',
    })
    data_dirs['train']
    data_dirs['test']

  For more customization on the download/extraction (ex: passwords, output_name,
  ...), you can give UrlInfo() or ExtractInfo() instead of plain string.

  It is also possible to use the download manager to process a custom function
  with the same guaranties/features than download and extraction.

    # The "/vocab/en-fr" path is cached and can be reused accros run. If
    # the path already exists, the cached value is re-used
    vocab_dir = dl_manager.execute_and_cache(generate_vocab_fn, '/vocab/en-fr')

  """

  def __init__(self, cache_dir, manual_dir=None, mode=None):
    """Download manager constructor.

    Args:
      cache_dir: `str`, Cache directory where all downloads, extractions and
        other artifacts are stored.
      manual_dir: `str`, Directory containing manually downloaded data. Default
        to cache_dir.
      mode (GenerateMode): Mode to FORCE_REDOWNLOAD, REUSE_CACHE_IF_EXISTS or
        REUSE_DATASET_IF_EXISTS. Default to REUSE_DATASET_IF_EXISTS.
    """
    self._cache_dir = os.path.expanduser(cache_dir)
    self._manual_dir = os.path.expanduser(manual_dir or cache_dir)
    self._backend = local_backend.LocalBackend()

    # The generation mode to indicates if we re-use the cached download or
    # force re-downloading data.
    mode = mode or util.GenerateMode.REUSE_DATASET_IF_EXISTS
    self._mode = util.GenerateMode(mode)  # str to Enum

    # Create the root directory if not exists yet
    gfile.MakeDirs(self._cache_dir)

  # Public API

  def download(self, urls_info):
    """Downloads the given urls.

    If one of the download already exists, the cached value will be reused.

    Args:
      urls_info (UrlInfo): The url to download. If a string is passed, it will
        automatically be converted into UrlInfo object

    Returns:
      downloaded_filepaths (str): The downloaded files paths
    """

    # We may want to add a pack kwarg such to keep all downloads in the same
    # directory

    def _download(url_info):
      """Function applied for each individual download."""
      # Convert string to UrlInfo object
      url_info = to_url_info(url_info)

      # Download the url
      return self._process_and_cache_uri(
          uri=url_info.url,
          uri_info=url_info,
          process_trial_fn=self._download,
      )

    # Run the download function on each of the urls
    return _parallel_run(
        _download,
        urls_info,
        max_workers=_NUM_PARALLEL_DOWNLOADS,
    )

  def extract(self, extracts_info):
    """Extract the given path.

    If one of the extraction already exists, the cached value will be reused.

    Args:
      extracts_info (ExtractInfo): The path to extract. If a string is passed,
        it will automatically be converted into ExtractInfo object

    Returns:
      extracted_dirs (str): The downloaded files
    """

    def _extract(extract_info):
      """Function applied for each individual extraction."""
      # Convert string to UrlInfo object
      extract_info = to_extract_info(extract_info)

      # Create the extract uri from the path
      if not extract_info.path.startswith(self._cache_dir):
        raise ValueError(
            'Trying to extract a file ({}) which is not in the cache of the '
            'download manager ({})'.format(extract_info.path, self._cache_dir))
      extract_uri = util.lchop(extract_info.path, self._cache_dir)
      # TODO(epot): There is a risk to reach the max length of filename (256
      # characters in linux and 1k total on CNS)
      extract_uri = os.path.join('extract://', extract_uri.strip('/'))

      # Process an extraction
      return self._process_and_cache_uri(
          uri=extract_uri,
          uri_info=extract_info,
          process_trial_fn=self._extract,
      )

    # Run the extract function on each of the paths
    return _parallel_run(
        _extract,
        extracts_info,
        max_workers=_NUM_PARALLEL_EXTRACTS,
    )

  def download_and_extract(self, urls_info):
    """Convinience method to perform download+extract in a single command.

    As most downloads are imediatelly followed by an extraction, this
    function avoid some boilerplate code. It is equivalent to:
      extracted_dirs = dl_manager.extract(dl_manager.download(urls_info))

    Args:
      urls_info (UrlInfo): Same input as .download()

    Returns:
      extracted_dir (str): Same output as .extract()
    """
    # Maybe the input could accept a UrlExtractInfo(url_info, extract_info)
    # which would contains both UrlInfo and ExtractInfo as currently only
    # the url can be controlled through the arguments.
    return self.extract(self.download(urls_info))

  def execute_and_cache(self, process_fn, cache_key):
    """Execute the function and cache the associated path.

    This function executes the process_fn using a custom cached directory given
    by the key. This allow to use the download manager to cache and share
    additional data like vocabulary or other intermediate artifacts.

    The key are shared between datasets which allow re-using data from other
    datasets but also increase the risk of name collision. A good practice
    is to add the dataset name as prefix of the key.

      vocab_cache_key = os.path.join(self.name, 'vocab/en-fr')
      vocab_dir = dl_manager.execute_and_cache(
          generate_vocab_fn,
          cache_key=vocab_cache_key,
      )

    Args:
      process_fn (fct): Function with signature "(cache_dir) -> None" which
        takes as input a directory on which storing the additional data.
      cache_key (str): Key of the cache to store the data. If the key already
        exists, the process_fn isn't used and the previously cached dir is
        reused.

    Returns:
      cache_dir (str): The directory on which the additional data has been
        written.
    """
    # TODO(epot): Replace this and extract uri by the urijoin ?
    cache_uri = os.path.join('local://', cache_key.strip('/'))

    def inner_process(trial):
      # Create the directory on which generate the additional data
      gfile.MakeDirs(trial.output_path)
      # Call the function to process the additional data
      return process_fn(trial.output_path)

    return self._process_and_cache_uri(
        uri=cache_uri,
        uri_info=None,
        process_trial_fn=inner_process,
    )

  @property
  def manual_dir(self):
    """Returns the directory containing the manually extracted data."""
    if not gfile.Exists(self._manual_dir):
      raise AssertionError(
          'Manual directory {} does not exist. Create it and download/extract'
          'dataset artifacts in there.'.format(self._manual_dir)
      )
    return self._manual_dir

  # Internal functions

  def _process_and_cache_uri(self, uri, uri_info, process_trial_fn):
    """Internal function which manage the cache and launch the trials.

    Args:
      uri (str): Uri to search for in the cache
      uri_info (obj): Additional info for the uri (UrlInfo or ExtractInfo), will
        be saved with the trial
      process_trial_fn (fct): Function with the signature "(UriTrial) -> None"
        which is called if the uri isn't found in the cache

    Returns:
      output_path (str): The output from the Trial
    """

    # Maybe should normalize the uri so that semantically equivalent urls
    # are detected as the same (ex: "http://a.io/" and "http://a.io") and
    # stored in the same register. Or issue guidelines about the url format.

    # Either get the previous cached location or build a new trial
    trial = self._get_or_create_trial(uri=uri, uri_info=uri_info)

    if trial.status != download_pb2.UriTrial.COMPLETED:
      with self._process_trial_controllers(trial):
        process_trial_fn(trial)

    return trial.output_path  # Return cached or processed trial

  def _get_or_create_trial(self, uri, uri_info=None):
    """Create a new trial or get the previous one.

    The previous trials are recreated by looking at the content of the cached
    dir.

    Args:
      uri (str): Uri to create the trial for.
      uri_info (obj): Object containing additional info about the download
        or extraction (UrlInfo or ExtractInfo).

    Returns:
      trial (UriTrial): Result of the trial containing the destination, status,
        timestamp,...
    """
    # The generation is deterministic so generating keys for the same uri will
    # always gives the same result
    trial_id = '{}_{}'.format(
        util.escape_uri(uri),
        util.hash_uri(uri),
    )

    log = util.build_log(prefix=trial_id)

    # Generate a new trial to eventually use
    trial = download_pb2.UriTrial(
        id=trial_id,
        status=download_pb2.UriTrial.IN_PROGRESS,
        output_path=os.path.join(self._cache_dir, trial_id),
    )
    add_uri_info(trial, uri, uri_info)

    if gfile.Exists(trial.output_path):

      # If the directory exists, the previous trial was complete (as it was
      # renamed successfully from ".incomplete")
      if self._mode == util.GenerateMode.FORCE_REDOWNLOAD:
        log('Cleanup previous trial: {}', trial.output_path)
        gfile.DeleteRecursively(trial.output_path)
      else:
        log('Reusing previously cached data...')
        # Try to reuse the previous download
        trial.status = download_pb2.UriTrial.COMPLETED

        # For the downloads, the output_path contains the file
        # TODO(epot): Should instead write the meta-data on disk (in a
        # ._trial.json) and replace ListDirectory() by a version which filter
        # the metadata file.
        is_dl = not any(uri.startswith(p) for p in ('local://', 'extract://'))
        is_gz = (
            uri.startswith('extract://') and
            uri.endswith('.gz') and
            not uri.endswith('tar.gz')
        )
        if is_dl or is_gz:
          trial.output_path = get_download_filepath(trial)
    else:
      log('No cached value found.')

    return trial

  def _process_trial_controllers(self, trial):
    """Decorators to apply before and after the process_trial_fn."""
    return use_incomplete_dir(trial)

  def _download(self, trial):
    """Downloads a single url given by the trial (thread safe).

    Args:
      trial (UriTrial): Object containing info about download.

    Raises:
      ValueError: If the destination dir is not empty
    """
    log = util.build_log(prefix=trial.id)

    # Check the download dir is empty
    if (gfile.Exists(trial.output_path) and
        gfile.ListDirectory(trial.output_path)):
      raise ValueError('Download dir {} should be empty'.format(
          trial.output_path))

    gfile.MakeDirs(trial.output_path)

    log('Start downloading...')
    self._backend.download(trial)

    # TODO(epot): Compute the checksum

    # Update the output path
    trial.output_path = get_download_filepath(trial)

    log('Download complete at {}', trial.output_path)

  def _extract(self, trial):
    """Extract a single file given by the trial.

    Args:
      trial (UriTrial): Object containing info about the extraction.

    Raises:
      ValueError: If the format is incorrect
    """
    log = util.build_log(prefix=trial.id)

    src = trial.extract_info.path
    dst = trial.output_path

    rar_format = ExtractFormat(ext=TAR_EXT, fn=self._backend.extract_tar)
    zip_format = ExtractFormat(ext=['.zip'], fn=self._backend.extract_zip)
    gz_format = ExtractFormat(ext=['.gz'], fn=self._backend.extract_gzip)
    # Order matter as '.tar.gz' will call _extract_tar while '.gz' will
    # call _extract_gzip
    extraction_fns = collections.OrderedDict([
        (download_pb2.ExtractInfo.RAR, rar_format),
        (download_pb2.ExtractInfo.ZIP, zip_format),
        (download_pb2.ExtractInfo.GZ, gz_format),
    ])

    # Filetype explicitly defined
    if trial.extract_info.filetype != download_pb2.ExtractInfo.UNKNOWN:
      extract_filetype = trial.extract_info.filetype
      extract_fn = extraction_fns[extract_filetype]
    # Try to infer the filetype from the name
    else:
      for extract_filetype, extract_format in extraction_fns.items():
        if any(src.lower().endswith(ext) for ext in extract_format.ext):
          extract_fn = extract_format.fn
          break
      else:  # No break (unrecognized archive)
        raise ValueError(
            'Unsuported archive file {} for trial {}. If you think this is an '
            'error, you can try to explicitly define the type in the '
            'ExtractFileType'.format(src, trial.id))

    log('Extract {} with {}...', src, extract_fn.__name__)
    gfile.MakeDirs(dst)
    extract_fn(src, dst)

    if extract_filetype == download_pb2.ExtractInfo.GZ:
      trial.output_path = get_download_filepath(trial)

def to_url_info(value):
  """Convert the strings into UrlInfo."""
  if isinstance(value, six.string_types):
    return download_pb2.UrlInfo(url=value)
  elif isinstance(value, download_pb2.UrlInfo):
    return value
  else:
    raise ValueError('Could not convert {} to UrlInfo'.format(value))

def to_extract_info(value):
  """Convert the strings into ExtractInfo."""
  if isinstance(value, six.string_types):
    return download_pb2.ExtractInfo(path=value)
  elif isinstance(value, download_pb2.ExtractInfo):
    return value
  else:
    raise ValueError('Could not convert {} to UrlInfo'.format(value))

def add_uri_info(trial, uri, uri_info):
  """Add the uri info to the UriTrial."""
  # Add the eventual info
  if uri.startswith('local://'):
    pass
  elif uri.startswith('extract://'):
    trial.extract_info.CopyFrom(uri_info)
  else:
    trial.url_info.CopyFrom(uri_info)

def get_download_filepath(trial):
  """Extract the downloaded file from the completed trial."""
  # Get the downloaded file:
  files = list(gfile.ListDirectory(trial.output_path))
  if not files:  # Should have been catched before, but just in case
    raise ValueError('Download {} failed!'.format(trial.id))
  elif len(files) > 1:
    raise ValueError('Multiple files detected for {}.'.format(trial.id))

  # Update the output path
  filename = files[0]
  return os.path.join(trial.output_path, filename)

@contextlib.contextmanager
def use_incomplete_dir(trial):
  """Wrap the trial in a temporary .incomplete path while it is processed."""
  # Replace the output dir by a temporary dir
  output_path_original = trial.output_path
  # Should add random string to avoid collision with local download manager ?
  output_path_tmp = trial.output_path + '.incomplete'
  trial.output_path = output_path_tmp
  yield
  if not trial.output_path.startswith(output_path_tmp):
    raise ValueError(
        'The output path for {} has been modified to {} and do not match '
        'the original {}'.format(trial.id, trial.output_path, output_path_tmp))
  gfile.Rename(output_path_tmp, output_path_original)
  output_path_extension = util.lchop(trial.output_path, output_path_tmp)
  trial.output_path = output_path_original + output_path_extension

def _parallel_run(function, input_struct, max_workers=1):
  """Run the function on each element of data_struct using a pool of workers."""

  # Distribute the work in a pool
  launch_thread_pool = concurrent.futures.ThreadPoolExecutor
  with launch_thread_pool(max_workers=max_workers) as executor:

    def launch_worker(value):
      return executor.submit(function, value)

    output_struct = utils.map_nested(launch_worker, input_struct)

  # Gather all results once all workers have finished
  def gather_results(value):
    return value.result()

  return utils.map_nested(gather_results, output_struct)


import json
import requests
import traceback

from .exceptions import APIResponseError

class PaginatedListResponse:
    """
    Iterable JSON API listing response with pagination

    By default django rest framework compatible. Override methods to fetch data
    from other types of responses.
    """

    def __init__(self, url, headers=dict, debug=False):
        self.headers = headers
        self.debug = debug

        self.__page_url__ = url
        self.__status_code__ = None
        self.__error_response__ = None
        self.__total_records__ = None
        self.__records__ = []

    def __len__(self):
        """
        Return total records in response

        If no data has been fetched, get first page to get total count
        """

        # Get first page to get count
        if not self.__records__ and self.__page_url__:
            self.__get_page__()

        return self.__total_records__

    def __get_page__(self):
        """
        Get next page for data

        Raises StopIteration if no more pages are available
        """

        if self.__page_url__ is None:
            raise StopIteration

        self.__records__ = []
        try:
            if self.debug:
                print('DEBUG get {}'.format(self.__page_url__))
            res = requests.get(self.__page_url__, headers=self.headers)
            self.__status_code__ = res.status_code
            if res.status_code == 200:
                data = json.loads(res.content)
                self.__records__ = self.get_records(data)
                self.__page_url__ = self.get_next_page_url(data)
                self.__total_records__ = self.get_total_records_count(data)
            else:
                self.__error_response__ = str(res.content, 'utf-8') if res.content else ''
                raise APIResponseError(
                    'Error getting {}: returns status code {}{}'.format(
                        self.__page_url__,
                        res.status_code,
                        ' ({})'.format(self.__error_response__) if self.__error_response__ else ''
                    )
                )
        except Exception as e:
            if self.debug:
                print(traceback.print_exc())
            raise APIResponseError('Error loading data: {}'.format(e))

    def __iter__(self):
        return self

    def __next__(self):
        """
        Get next record from iterator.

        Loads next page as side effect if available, until all pages are loaded
        """

        if self.__records__:
            # Got records to return, pop one and return it
            return self.__records__.pop(0)

        elif self.__page_url__:
            # Got a page URL to load, load page and pop first record from loaded response
            self.__get_page__()
            if self.__records__:
                return self.__records__.pop(0)
            else:
                raise StopIteration
        else:
            raise StopIteration

    def get_records(self, data):
        """
        Get records from data

        By default look up 'records' attribute (django rest framework compatible)
        """
        if isinstance(data, dict):
            if 'results' in data:
                return data.get('results', [])
            else:
                return [data]
        elif isinstance(data, list):
            return data
        else:
            raise ValueError('Unexpected data type: {}'.type(data))

    def get_next_page_url(self, data):
        """
        Get next page's URL from data

        By default look up 'next' field (django rest framework compatible)
        """
        if isinstance(data, dict):
            return data.get('next', None)
        return None

    def get_total_records_count(self, data):
        """
        Get count of total records

        By default look up 'count' field (django rest framework compatible)
        """
        if isinstance(data, dict):
            return data.get('count', None)
        elif isinstance(data, list):
            return len(data)
        else:
            return None


from django.contrib import admin
from .models import *
from django.utils.translation import ugettext_lazy as _

@admin.register(Student)
class StudentAdmin(admin.ModelAdmin):
    fields = ('s_number', 's_name', 's_surname', 's_education', 's_is_dgs', 'dgs_file')
    list_display = (
        's_number', 's_name', 's_surname', 's_education', 's_is_dgs'
    )
    list_filter = ('s_number', 's_education')
    search_fields = ('s_number', 's_name ', 'company_name')

@admin.register(Internship)
class InternshipAdmin(admin.ModelAdmin):
    fields = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    list_display = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    list_filter = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    search_fields = ('i_student__s_number',)

@admin.register(Commission)
class CommissionAdmin(admin.ModelAdmin):
    fields = ('c_name', 'c_surname', 'c_internship')
    list_display = ('get_commission', 'c_surname')
    list_filter = ('c_name', 'c_surname')
    search_fields = ('c_name', 'c_surname')

    def get_commission(self, obj):
        commission = '-'.join(
            [commission.c_name for commission in obj.commission.all()]
        )
        return commission

@admin.register(Interview)
class InterviewAdmin(admin.ModelAdmin):
    fields = ('internship', 'commission', 'i_datetime')
    list_display = ('internship', 'i_datetime', 'get_commission')
    list_filter = ('internship', 'commission', 'i_datetime')
    search_fields = ('internship', 'commission', 'i_datetime')

    def get_commission(self, obj):
        commission = '-'.join(
            [commission.c_name for commission in obj.commission.all()]
        )
        return commission

@admin.register(NoteScale)
class NoteScaleAdmin(admin.ModelAdmin):
    fields = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations',
        'content', 'interview'
    )
    list_display = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )
    list_filter = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )
    search_fields = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )

    def get_n_internship(self, obj):
        n_internship = '-'.join(
            [n_internship.interview for n_internship in obj.n_internship.all()]
        )
        return n_internship


__author__ = "Laurine"

# Question
# Let’s print a triangle made of asterisks (‘*’) separated by spaces. The triangle
# should consist of n rows, where n is a given positive integer, and consecutive rows should
# contain 1, 2, . . . , n asterisks. For example, for n = 4 the triangle should appear as follows:

# Specify the number of rows
rows = 5

for i in range(1, rows+1):
    # for every row, print an arterisk and a space multiplied by the row number
    print("* " *i)


import requests
import bs4
from bs4 import BeautifulSoup
import re
import os
import traceback

def getCurrInfo(fpath):
    try:
        bocurl = 'http://www.usd-cny.com/bankofchina.htm'
        html = getHTML(bocurl)
        infoDict = {}
        soup = BeautifulSoup(html, 'html.parser')
        currInfo = soup.find('div', attrs={'class': 'pp'})
        name = currInfo.find_all('a')[0]
        infoDict.update({'»õ±ÒÃû³Æ': name.text.split()[0]})

        keyList = currInfo.find_all('th')
        valueList = currInfo.find_all('td')
        keyList1 = currInfo.find_all('a')
        print('count country', len(keyList1))
        for i in range(len(keyList1)):
            key = keyList1[i].text
            val = valueList[i].text
            infoDict[key] = val

        with open(fpath, 'w', encoding='utf-8') as f:
            f.write(str(infoDict) + '\n')
    except:
        traceback.print_exc()


"""Test module for the `map_columns` processor."""

from copy import deepcopy
from pytest import fixture
from common.processors.map_columns import build_lookup_table, apply_mapping

@fixture(scope='function')
def lookup_table():
    """The lookup table is a generator so we need a new one for each test.
    """
    datapackage = {
        'resources': [
            {
                'schema': {
                    'fields': [
                        {
                            'name': 'foo',
                            'maps_to': 'project_id'
                        },
                        {
                            'name': 'bar',
                            'maps_to': 'invalid_fiscal_field'
                        },
                        {
                            'name': 'spam',
                            'maps_to': None
                        },
                        {
                            'name': 'eggs',
                        },

                    ]

                }
            }
        ]
    }
    return build_lookup_table(datapackage)

@fixture(scope='function')
def input_row():
    """Row objects are mutable so we need a new one for each test.
    """
    return {
        'foo': 1,
        'bar': 2,
        'spam': 3,
        'eggs': 4
    }

# noinspection PyShadowingNames
def test_build_lookup_table_returns_correct_mapping(lookup_table):
    assert dict(lookup_table) == {'foo': 'project_id'}

# noinspection PyShadowingNames
def test_apply_mapping_toggles_the_correct_fields(input_row, lookup_table):
    output_row = apply_mapping(deepcopy(input_row), mapping=lookup_table)
    assert set(output_row) ^ set(input_row) == {'project_id', 'foo'}

# noinspection PyShadowingNames
def test_apply_mapping_returns_correct_row(input_row, lookup_table):
    expected_output = {
        'project_id': 1,
        'bar': 2,
        'spam': 3,
        'eggs': 4
    }
    assert apply_mapping(input_row, mapping=lookup_table) == expected_output

# noinspection PyShadowingNames
def test_apply_mapping_with_empty_lookup_returns_identical_row(input_row):
    assert apply_mapping(input_row, mapping=None) == input_row


import csv
from pyExcelerator import *

def read_csv(filename):
    with open(filename+".csv") as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            process_row(row)

def process_row(row):
    print(row)
    name_parts = row['Teacher'].split(' ', 1)
    name_form = name_parts[0].replace('.', '')
    name = name_parts[1]
    room = row['Room']
    grade = row['Grade']
    sheet_name = name_form + '_' + name + '_' + grade + '_' + room
    filename = "teachers/" + sheet_name + ".xls"
    print(filename)
    workbook = Workbook()
    worksheet = workbook.add_sheet(sheet_name)
    worksheet.write(0, 0, 'First Name')
    worksheet.write(0, 1, 'Last Name')
    worksheet.write(0, 2, 'Project Type')
    worksheet.write(0, 3, 'Project Title')
    worksheet.write(0, 4, 'Project Description')
    worksheet.write(0,10, '('+sheet_name+')')
    workbook.save(filename)

read_csv("teachers")


from django.core.urlresolvers import reverse
from rest_framework.status import HTTP_200_OK, HTTP_204_NO_CONTENT, \
    HTTP_401_UNAUTHORIZED
from rest_framework.test import APITestCase

from users.tests.factories import UserFactory

class UserViewTestCase(APITestCase):
    """Tests for UserView."""

    def setUp(self):
        self.api_url = reverse('api:users:user')

    def test_self_view_for_not_authenticated_user(self):
        response = self.client.get(self.api_url, format='json')

        self.assertEqual(response.status_code, HTTP_401_UNAUTHORIZED)
        self.assertEqual(response.data['detail'],
                         'Authentication credentials were not provided.')

    def test_self_view_for_authenticated_user(self):
        user = UserFactory()
        self.client.force_authenticate(user=user)

        response = self.client.get(self.api_url, format='json')

        self.assertEqual(response.status_code, HTTP_200_OK)
        self.assertEqual(response.data['email'],
                         user.email)
        self.assertEqual(response.data['full_name'],
                         user.full_name)

class PublicUserViewTestCase(APITestCase):
    """Tests for PublicUserView."""

    def test_for_not_public_user_profile(self):
        private_user = UserFactory()
        api_url = reverse('api:users:public-user',
                          kwargs={'user_id': private_user.pk})

        response = self.client.get(api_url, format='json')
        self.assertEqual(response.status_code, HTTP_204_NO_CONTENT)

    def test_for_public_user_profile(self):
        public_user = UserFactory(is_public=True)
        api_url = reverse('api:users:public-user',
                          kwargs={'user_id': public_user.pk})

        response = self.client.get(api_url, format='json')
        self.assertEqual(response.status_code, HTTP_200_OK)
        self.assertEqual(response.data['email'],
                         public_user.email)
        self.assertEqual(response.data['full_name'],
                         public_user.full_name)