# 通常我们使用误差平方和，即SSE(Sum of Squared Error)来度量聚类效果的指标，SSE值越小表示数据点越接近于它们的质心，聚类效果好
# 如何改进聚类的效果，一种方法是将具有最大SSE值的簇分成两个，将最大簇包含的点过滤出来并在这些点上运行K-均值算法。这样的代价是需要把两个簇进行合并。
# 上述的方法在低维的时候可以使用，对于高维的情况，计算所有质点的之间的距离，然后合并距离最近的两个点；第二种方法是合并两个簇，计算总SSE值，在所有的两个簇上重复知道找到合并最佳的两个簇位置。
# 下面我们将讨论二分K-均值算法，一种克服均值算法收敛到局部最优解的方法。思路是将有点视为一个簇，然后将簇一分为二。之后选择一个簇继续进行划分，选择哪一个簇进行划分取决于划分后是否可以最大程度降低SSE值。
# 上述基于SSE划分的过程不断重复直到划分k个簇。

import numpy as np

def loadDataSet(filename):
	dataMat = []
	fr = open(filename)
	for line in fr.readlines():
		curLine = line.split()
		fltLine = [float(word) for word in curLine]
		dataMat.append(fltLine)
	fr.close()
	return dataMat

def calcudist(vecA,vecB):
	return np.sqrt(np.sum(np.power(vecA - vecB,2)))

def randCent(dataSet,k):
	n = np.shape(dataSet)[1]
	centerpoint = np.mat(np.zeros((k,n)))
	for j in range(n): # 求每一列的结果
		minJ = min(dataSet[:,j])
		rangeJ = float(max(dataSet[:,j])-minJ)
		centerpoint[:,j] = minJ + rangeJ * np.random.rand(k,1)
	return centerpoint

def KMeans(dataSet,k,distMeas = calcudist,createCent = randCent):
	m = np.shape(dataSet)[0]
	clusterAssment  = np.mat(np.zeros((m,2)))
	centerpoint = createCent(dataSet,k)
	clusterChanged = True
	while clusterChanged:
		clusterChanged = False
		for i in range(m):
			minDist = float('inf')
			minIndex = -1
			for j in range(k):
				distIJ = distMeas(centerpoint[j,:],dataSet[i,:])
				if distIJ<minDist:
					minDist = distIJ
					minIndex = j
			if clusterAssment[i,0] != minIndex:
				clusterChanged = True
			clusterAssment[i,:] = minIndex,minDist**2
		print(centerpoint)
		for cent in range(k):
			Sameclu = dataSet[np.nonzero(clusterAssment[:,0].A == cent)[0]]
			centpoint[cent,:] = np.mean(Sameclu,axis=0)
	return centerpoint,clusterAssment

def biKmeans(dataSet,k,distMeas = calcudist):
	m = np.shape(dataSet)[0] # 返回数据的总数
	clusterAssment = np.mat(np.zeros((m,2)))  # 构建一个综述的矩阵
	# centerpoint = np.mean(dataSet,axis = 0).tolist()[0]  # 将数组数据的副本作为list返回，这里取list的第一个数据，就是一组数，因为tolist之后返回的是[[-0.111，-0.111]]这种类型的数据
	# cenList = [centerpoint] # 这句和上面的一句可以直接合并
	centList = np.mean(dataSet,axis = 0).tolist() # 得到所有质心的均值，即第一个簇
	# centList用来保留所有的质心
	centerpoint = centList[0]
	# 这里取出第一个质心，刚开始只有初始簇
	for j in range(m): # 计算所有数据点和第一个质心的距离误差
		clusterAssment[j,1] = distMeas(np.mat(centerpoint),dataSet[j,:])**2
	# 开始循环划分簇
	while(len(centList)<k): #小于需要划分的簇的数量
		lowestSSE = float('inf') # 误差平方和设置为无穷大
		for i in range(len(centList)): #遍历已经划分的簇，即所有的质心
			ptsInCurrCluster = dataSet[np.nonzero(clusterAssment[:,0].A==i),:] # 找出第一个簇的质心，即该簇的所有数据点。这里返回80×2
			centerMat,splitClustAss = KMeans(ptsInCurrCluster,2,distMeas) # 把第一个簇划分成两个簇
			sseSplit = np.sum(splitClustAss[:,1]) # 返回两个簇的总误差
			sseNotSplit = np.sum(clusterAssment[np.nonzero(clusterAssment[:,0].A != i)[0],1])# .A将矩阵转换成numpy的array数组，剩下的数据点的总误差
			print("sseSplit, and notSplit:",sseSplit,sseNotSplit)
			if (sseSplit + sseNotSplit) < lowestSSE: # 比较划分之后的总误差和最小误差，如果这次划分得到的误差结果减小，则表明划分
				bestCentToSplit = i # 这次划分的簇类型
				bestNewCents = centerMat # 新的质心
				bestClustAss = splitClustAss.copy() # 分配结果的副本
				lowestSSE =  sseSplit + sseNotSplit
		#前面的for循环决定了这次的2分划分保存，下面就是更新，通过过滤器更新
		# 首先从划分结果里面找到0,1的数据点，将其所述的分类，首先将簇分配结果中等于1的划分为centList的长度，等于0的则为
		bestClustAss[np.nonzero(bestClustAss[:,0].A==1)[0],0]= len(centList)
		# 等于0 的则划分到当前的簇分类中，即i
		bestClustAss[np.nonzero(bestClustAss[:,0].A==0)[0],0] = bestCentToSplit
		print("the bestCentToSplit is : ",bestCentToSplit)
		print("the len of bestClustAss is : ",len(bestClustAss))
		cenList[bestCentToSplit] = bestNewCents[0,:] # 当前质心更新为新划分的两个簇的一个质心
		cenList.append(bestNewCents[1,:]) # 另外一个划分的质心则新加入到centList中
		clusterAssment[np.nonzero(clusterAssment[:,0].A==bestCentToSplit)[0],:] = bestClustAss # 把新划分的簇的下标更新
	return np.mat(cenList),clusterAssment

def picResult(myCentpoint,clusterAssing,dataSet):
	X = np.array(myCentpoint[:,0]) # 选择质心的x坐标,matrix有约束，所以需要转化成numpy的数组才好画散点图
	Y = np.array(myCentpoint[:,1]) # 选择质心的y坐标
	fig = plt.figure()
	ax1 = fig.add_subplot(111)
	ax1.set_title('Scatter Plot')
	plt.xlabel('X')
	plt.ylabel('Y')
	ax1.scatter(X,Y,c='r',marker= '+',s=300)

	scatterMarkers=['s', 'o', '^', '8', 'p', \
                    'd', 'v', 'h', '>', '<']
	for i in range(4):
		ptsInCurrCluster = dataSet[np.nonzero(clusterAssing[:,0].A==i)[0],:]
		markerStyle = scatterMarkers[i%len(scatterMarkers)]
		ax1.scatter(ptsInCurrCluster[:,0].flatten().A[0],ptsInCurrCluster[:,1].flatten().A[0],marker = markerStyle,s=50)
	plt.show()

if __name__ == '__main__':
	dataMat = np.mat(loadDataSet("testSet.txt"))
	myCentpoint,clusterAssing = biKmeans(dataMat,4)
	print("质心为：\n",myCentpoint)
	print("聚类结果为：\n",clusterAssing)
	picResult(myCentpoint,clusterAssing,dataMat)

K-均值聚类/k均值聚类3.py


#!/usr/bin/env python3
# 2017/01/30

# input
n = 3
k = 2
wv = [(2, 2), (5, 3), (2, 1)]

# solution
def C(x, n, k, wv):
    y = []
    for w, v in wv:
        y.append(v - x * w)
    y = sorted(y)[::-1]
    s = 0
    for i in range(k):
        s += y[i]
    return s >= 0

def solve(n, k, wv):
    left = 0
    right = 10 ** 10
    for i in range(100):
        mid = (left + right) / 2
        if C(mid, n, k, wv):
            left = mid
        else:
            right = mid
    return format(left, '.2f')

def test():
    assert solve(n, k, wv) == '0.75'
    return 'test pass'

if __name__ == '__main__':
    print(test())


import config
import smtplib

def SendMail(token, list):
    try:
        with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
            smtp.login(config.EMAIL, config.PASSWORD)
            if token == 1:
                subject = 'Grabber: Spreadsheet Updated'
                body = f'Successfully grabbed all items!\nYour total is $ {list[0]}'
                msg = Message(subject, body)
            elif token == 2:
                subject = 'Grabber: Price Dropped!!'
                body = f'The price of "{list[0]}" has dropped from {list[1]} to {list[2]}'
                msg = Message(subject, body)
            elif token == 3:
                subject = 'Grabber: Product Purchased'
                body = f'The requested product has dropped to sale from {list[0]} to {list[1]} and was purchased.'
                msg = Message(subject, body)
            smtp.sendmail(config.EMAIL, config.TO, msg)
        print('Email sent scuccessfully')
    except:
        print('Email failed to send!')

def Message(subj, body):
    return (f'Subject: {subj}\n\n{body}')


import cv2

for i in dir (cv2):
    if 'video' in i or 'Video' in i:
                 print (i)

#starting cam
x=cv2.VideoCapture(0)

width=int(x.get(cv2.CAP_PROP_FRAME_WIDTH))
height=int(x.get(cv2.CAP_PROP_FRAME_HEIGHT))

#XVID for .avi,,,deciding format
video_format=cv2.VideoWriter_fourcc(*'XVID')
#saving data
video_output=cv2.VideoWriter('ad.avi',video_format,20.0,(width,height))

for i in dir (cv2):
    if 'width' in i or 'HEIGHT' in i:
                 print (i)

while x.isOpened():
    status,frame=x.read()
    cv2.imshow("hello",frame)
    video_output.write(frame)

    if cv2.waitKey(1) & 0xFF==ord('q'):
        break
cv2.destroyAllWindows()
x.release()


#to do: missing indicator, mvp
from sklearn import preprocessing
from scipy import stats
import numpy as np
import pandas as pd
import math
import gpxpy.geo

# check skewness for all numerical variables(including price) and log transform skewed numeric features
def log_skewness(data, num_cols, Y):

    #numeric_feats = data.dtypes[data.dtypes != "object"].index
    #skewed_feats = data[num_cols].apply(lambda x: stats.skew(x))  # compute skewness
    #skewed_feats = skewed_feats[skewed_feats > 0.8]
    #skewed_feats = skewed_feats.index
    #data[skewed_feats] = np.log1p(data[skewed_feats])

    data[Y] = np.log(data[Y]) # log(price)

    return data

# detect outliers of target price and delete outliers
# outliers are price which outside 2 standard deviations(or 1.96) # 2.575
def del_outliers(data, Y='price'):

    outliers = []
    data_array = np.array(data[Y])
    mean = np.mean(data_array)
    std = np.std(data_array)

    for i in range(0, len(data_array)):
        if abs(data_array[i] - mean) > 1.96 * std:
            outliers.append(data_array[i])
    thred = sorted(outliers)[0]
    data[Y][data[Y] > thred] = np.nan
    data = data.dropna()

    return data

# standardization
def standardize_matrix(matrix):

    for col in range(matrix.shape[1]):
        mean = np.mean(matrix[:,col])
        std = np.std(matrix[:,col])
        matrix[:, col] = list(map(lambda x: (x - mean) / std, matrix[:,col]))

    return matrix

def standardize_df(data):

    for col in data.columns.tolist():
        if col != 'price': # not standardize target
            mean = np.mean(data[col])
            std = np.std(data[col] )
            data[col] = data[col].apply(lambda x: (x - mean) / std)

    return data

# normalization
def normalize_df(data):

    for col in data.columns.tolist():
        if col != 'price': # not standardize target
            data_col_max, data_col_min = [data[col].max(axis=0), data[col].min(axis=0)]
            data[col] = data[col].apply(lambda x: x - data_col_min) / (data_col_max - data_col_min)

    return data

# create feature: distance to subway

def prepare_subway_data():
    '''
    data source: https://data.ny.gov/Transportation/NYC-Transit-Subway-Entrance-And-Exit-Data/i9wp-a4ja
    '''
    # read external subway dataset
    subway = pd.read_csv('./data/subway.csv')
    useful_column = ['Station Longitude', 'Station Latitude', 'Station Name']
    subway = subway[useful_column]
    subway = subway.dropna()
    subway = subway.drop_duplicates()
    subway = subway.reset_index(drop=True)

    return (subway)

def prepare_park_data():
    '''
    data source: https://data.cityofnewyork.us/Recreation/NYC-Parks-Public-Events-Upcoming-14-Days/w3wp-dpdi
    '''
    # read external park dataset
    park = pd.read_json('./data/park.json')
    park = pd.DataFrame(park)
    useful_column = ['parknames', 'coordinates']
    park = park[useful_column]

    # clean park dataset
    park = park.drop_duplicates()
    park = park.replace(r'', np.nan, regex=True)
    park = park.dropna()
    park['latitude'], park['longitude'] = park['coordinates'].str.split(', ', 1).str
    park = park.drop('coordinates', 1)
    # park[park['longitude'].str.contains(";")] # row 77, 121, 123 are anomalous
    park.drop(park.index[[77, 121, 123]], inplace=True)
    park = park.reset_index(drop=True)
    park['latitude'] = park['latitude'].astype('float64')
    park['longitude'] = park['longitude'].astype('float64')

    return (park)

def create_new_feature(data):
    '''
    subway feature created: count_near_subway, dist_to_nearest_subway
    park feature created: count_near_park, dist_to_nearest_park
    '''

    park = prepare_park_data()
    subway = prepare_subway_data()

    data = data.reset_index(drop=True)

    # create two subway features
    data['count_near_subway'] = np.zeros(len(data))
    data['dist_to_nearest_subway'] = np.zeros(len(data))
    data['dist_to_nearest_subway'] = 1600

    # create two park features
    data['count_near_park'] = np.zeros(len(data))
    data['dist_to_nearest_park'] = np.zeros(len(data))
    data['dist_to_nearest_park'] = 1600

    # set up threshold, point to point distance, unit: meters (i.e. 0.25 miles)
    dist_to_subway = 400
    dist_to_park = 800

    for i in range(data.shape[0]):
        # house location
        lat1 = data['latitude'][i]
        lon1 = data['longitude'][i]
        min_dist12 = 1600
        min_dist13 = 1600
        print (i)
        for j in range(subway.shape[0]):
            # subway location
            lat2 = subway['Station Latitude'][j]
            lon2 = subway['Station Longitude'][j]

            # args order is super important
            dist12 = gpxpy.geo.haversine_distance(lat1, lon1, lat2, lon2)

            if dist12 <= dist_to_subway:
                data['count_near_subway'][i] += 1

            if dist12 < min_dist12:
                data['dist_to_nearest_subway'][i] = dist12
                min_dist12 = dist12

            try:
                lat3 = park['latitude'][j]
                lon3 = park['longitude'][j]
                dist13 = gpxpy.geo.haversine_distance(lat1, lon1, lat3, lon3)
                if dist13 <= dist_to_park:
                    data['count_near_park'][i] += 1

                if dist13 < min_dist13:
                    data['dist_to_nearest_park'][i] = dist13
                    min_dist13 = dist13
            except KeyError:
                continue

    return (data)

'''
datapath = './data/encoded_others.pkl'
pkl_file = open(datapath, 'rb')
dataset = pickle.load(pkl_file)
'''


import unittest
import subprocess
from elephant import (
main,
canvas,
get_canvas
)

class ElephantTests(unittest.TestCase):
	"""Test class to test elephant.py functionality"""
	def setUp(self):
		canvas(['30', '5'],'test.txt')

	def tearDown(self):
		pass

	def test_get_canvas(self):
		res = get_canvas('test.txt')
		self.assertEqual(res, (32, 236, 5))

	def test_get_canvas_error(self):
		with self.assertRaises(FileNotFoundError):
			get_canvas('output.txt')

	def test_get_canvas_value_error(self):
		f = open('out.txt', 'w')
		with self.assertRaises(ValueError):
			get_canvas('out.txt')
		f.close()

if __name__ == '__main__':
	unittest.main()


"""
keys() and values()

While .items() returns an array of tuples with each tuple consisting of a key/value pair from the dictionary:

    The .keys() method returns a list of the dictionary’s keys, and
    The .values() method returns a list of the dictionary’s values.

Again, these methods will not return the keys or values from the dictionary in any specific order.

You can think of a tuple as an immutable (that is, unchangeable) list. Tuples are surrounded by ()s and can contain any data type.

"""

my_dict = {'name':'sebastian',
           'lastName':'henao',
           'nikName':'sebas'
          }
#print(my_dict.items())
print(my_dict.keys())
print(my_dict.values())

for key in my_dict:
  print(key, my_dict[key])


#! /usr/bin/env python

# This script prepares ESM output with LCLM chessboard pattern
# to separate into local, nonlocal, and total effects

# It is based on the scripts of Johannes Winckler for his 2017 study
# doi: 10.1175/JCLI-D-16-0067.1

# This function takes a difference map the variable name and the interpolation method.

# Calculate everything on extended grid, such that values around zero meridian can be interpolated correctly.
# Output are interpolated local, non-local, and total effects.

# run the script as: python prepare_local_nonlocal_total_effects.py FILE_DIFFERENCE_MAP VARIABLE_NAME INTERPOLATION_METHOD

import numpy as np
#from scipy.interpolate import RectBivariateSpline
import scipy.interpolate as interpolate
import datetime as dt
import os
from copy import deepcopy as cp
import netCDF4 as nc
import sys
from dask import delayed
import pandas as pd
import xarray as xr
import time
from collections import Counter
import matplotlib.pyplot as plt
import pickle

# INTERPOLATION METHOD = "interpolate_total" or "interpolate_local"

# GET INFO FROM INPUT
if os.path.isfile(sys.argv[1]):
    FILE_DIFFERENCEMAP = sys.argv[1]
    if "interpolate" in sys.argv[2]:
        INTERPOLATE_METHOD = sys.argv[2]
        VAR = sys.argv[3]
    else:
        INTERPOLATE_METHOD = sys.argv[3]
        VAR = sys.argv[2]

elif os.path.isfile(sys.argv[2]):
    FILE_DIFFERENCEMAP = sys.argv[2]
    if "interpolate" in sys.argv[1]:
        INTERPOLATE_METHOD = sys.argv[1]
        VAR = sys.argv[3]
    else:
        INTERPOLATE_METHOD = sys.argv[3]
        VAR = sys.argv[1]

elif os.path.isfile(sys.argv[3]):
    FILE_DIFFERENCEMAP = sys.argv[3]
    if "interpolate" in sys.argv[2]:
        INTERPOLATE_METHOD = sys.argv[2]
        VAR = sys.argv[1]
    else:
        INTERPOLATE_METHOD = sys.argv[1]
        VAR = sys.argv[2]

else:
    print(sys.argv[:], " there is no file or wrong interpolation method in the parsed arguments...")
    sys.exit( 2)

# GET INFO ABOUT THE MODEL
if "cesm" in FILE_DIFFERENCEMAP:
    MODEL = "cesm"
    FILL_VALUE = 1e36
elif "mpiesm" in FILE_DIFFERENCEMAP:
    MODEL = "mpiesm"
    FILL_VALUE = 1e20
elif "ecearth" in FILE_DIFFERENCEMAP:
    MODEL = "ecearth"
    FILL_VALUE=9.96921e36
else:
    print("The model could not be identified... :|")
    sys.exit( 2)

#INPUT_WD = "/home/felixh/data/mpiesm/chessboard/"
INPUT_WD = "/pf/b/b380948/lamaclima/chessboard_tests/"
DATA_WD = "/home/felixh/data/mpiesm/differencemaps/ctl-crop/"

# IMPORT GLOBAL CHESSBOARD DISTRIBUTION ON LAND
#ifile = INPUT_WD + "lamaclima_experiments_chessboard_pattern_no-missing.nc"
if MODEL != "ecearth":   ##data with a regular grid
    ifile = INPUT_WD + "lamaclima_experiments_chessboard_pattern_" + MODEL + ".nc"
    f = nc.Dataset(ifile, 'r')
    CHESSBOARD = f.variables['chessboard_pattern'][:]
    LAT = f.variables['lat'][:]
    LON = f.variables['lon'][:]
    f.close()
    # IMPORT GLOBAL CHESSBOARD DISTRIBUTION GLOBALLY
    ifile = INPUT_WD + "lamaclima_experiments_global_chessboard_pattern_" + MODEL + ".nc"
    f = nc.Dataset(ifile, 'r')
    GLOBAL_CHESSBOARD = f.variables['chessboard_pattern'][:]
    f.close()

else:
    CHESSBOARD_LAT=pd.read_csv('/pf/b/b381334/files_ec_earth/Chessboard_normal_ctrl_grids_info/EC-Earth_CTRL_Map_Lat.csv')
    CHESSBOARD_LON=pd.read_csv('/pf/b/b381334/files_ec_earth/Chessboard_normal_ctrl_grids_info/EC-Earth_CTRL_Map_Lon.csv')
    CHESSBOARD_LAT=CHESSBOARD_LAT.iloc[:,1].values
    CHESSBOARD_LON=CHESSBOARD_LON.iloc[:,1].values

    ifile = "/pf/b/b381334/files_ec_earth/ec-earth_land_sea_mask.nc"
    f = nc.Dataset(ifile, 'r')
    #SEA_LAND_MASK = f.variables['LANDFRAC_PFT'][:]
    SEA_LAND_MASK = f.variables['var172'][:] * 1.0
    GLACIER = f.variables['var172'][:] * 0.0  # no glacier mask provided for ecearth so I just give the land sea mask again
    #CHECKERS=np.ones(len(LAT))
    #print(len(LAT))
    #print(len(CHESSBOARD_LAT))
    #for i in range(len(LAT)):
    #    for j in range(len(CHESSBOARD_LAT)):
            #print([np.round(LAT[i],3),np.round(LON[i],3)],[np.round(CHESSBOARD_LAT[j],3),np.round(CHESSBOARD_LON[j],3)])
    #        if [np.round(LAT[i],3),np.round(LON[i],3)]==[np.round(CHESSBOARD_LAT[j],3),np.round(CHESSBOARD_LON[j],3)]:
    #           CHECKERS[i]=1
               #print('chess')
    #           i=i+1
    #        else:
    #           CHECKERS[i]=0
            #time.sleep(0.1)
    #print(CHECKERS)
    #da= xr.DataArray(data=CHECKERS,dims=["space"],coords=dict(space=(["space"], LAT)),attrs=dict(description="Checkerboard pattern EC-EARTH"))
    #da.to_netcdf("/scratch/b/b381334/files_ec_earth/Chessboard_normal_ctrl_grids_info/lamaclima_experiments_global_chessboard_pattern_ecearth.nc")
    # IMPORT CHESSBOARD DISTRIBUTION (only over land for ecearth)
    ifile = "/pf/b/b381334/files_ec_earth/Chessboard_normal_ctrl_grids_info/lamaclima_experiments_global_chessboard_pattern_ecearth.nc"
    f = nc.Dataset(ifile, 'r')
    CHESSBOARD = f.variables['__xarray_dataarray_variable__'][:]

    #read in grid data (only lat defined, lon needs to be deduced based on spatial resolution definition
    grid=pd.read_csv('/pf/b/b381334/files_ec_earth/Chessboard_normal_ctrl_grids_info/ec_earth_grid_info.csv', thousands=',')
    lat_grid=grid.iloc[3:,3].str.replace(',', '').astype(float).div(1000000).reset_index(drop=True).values
    rpoints=grid.iloc[3:,1]
    delta_lon=rpoints.astype(float).div(360)
    delta_lon=delta_lon.pow(-1).reset_index(drop=True).values
    ## make global LAT and LON arrays
    tel_lat=0
    tel=1
    LAT=np.zeros((len(SEA_LAND_MASK[0,:]),1))
    LON=np.zeros((len(SEA_LAND_MASK[0,:]),1))
    tel_lat_old=20 #value gets overwritten during loop just placeholder
    for i in range(len(SEA_LAND_MASK[0,:])):
        tel=tel+1
        delta=delta_lon[[tel_lat]]
        LAT[i]= lat_grid[[tel_lat]]
        if (LAT[i]<1 and LAT[i]>-1):
            LAT[i]=LAT[i]*1000000    #these have been read in wrong so need to be corrected
        #time.sleep(0.5)
        if tel_lat !=tel_lat_old:  ##check if first instance of lat band, start at lon=0
            LON[i]=0
        else:
            LON[i]=LON[i-1]+delta
        tel_lat_old=tel_lat
        if int(360/delta)==int(tel-1):
            tel_lat=tel_lat+1
            tel=1
    ##note: I checked this calculation against the real lat and lon and it checks out

    for i in range(len(SEA_LAND_MASK[0,:])):
        if LON[i]>180:
            LON[i]=LON[i]-360 #make lon range from -180 to 180
    #f.close()
    #saving these files so I can access them more quickly when needed
    pickle.dump(LAT, open('/pf/b/b381334/lat_ec_earth.sav', 'wb'))
    pickle.dump(LON, open('/pf/b/b381334/lon_ec_earth.sav', 'wb'))
    pickle.dump(CHESSBOARD_LAT, open('/pf/b/b381334/checkers_lat_ec_earth.sav', 'wb'))
    pickle.dump(CHESSBOARD_LON, open('/pf/b/b381334/checkers_lon_ec_earth.sav', 'wb'))

# IMPORT SEA LAND MASK (except EC-EARTH)
if MODEL == "mpiesm":
    ifile = INPUT_WD + "jsbach_T63GR15_11tiles_5layers_2005_dynveg_slm_glac.nc"
    f = nc.Dataset(ifile, 'r')
    SEA_LAND_MASK = f.variables['slm'][:]
    GLACIER = f.variables['glac'][:]
    f.close()

elif MODEL == "cesm":
    ifile = INPUT_WD + "landmask_glacier_cesm.nc"
    f = nc.Dataset(ifile, 'r')
    #SEA_LAND_MASK = f.variables['LANDFRAC_PFT'][:]
    SEA_LAND_MASK = f.variables['landmask'][:] * 1.0
    GLACIER = f.variables['GLACIER_REGION'][:] * 1.0 # simple transformation to float...
    f.close()

# IMPORT DIFFERENCEMAP
#ifile = DATA_WD + "nep_Emon_MPI-ESM1-2-LR_215501-217412_ctl-crop_timavg.nc"
ifile = FILE_DIFFERENCEMAP
f = nc.Dataset(ifile, 'r+')
# CHECK IF INVERTED LAT VALUES IN ORIGINAL FILE AND KEEP THE ORIGINAL CONVENTION

if MODEL != 'ecearth':
   LAT_DIFF = f.variables["lat"][:]
   LON_DIFF = f.variables["lon"][:]
else:
   LAT_DIFF=LAT[:]
   LON_DIFF=LON[:]

if np.around( LAT_DIFF[0], 5) == np.around( LAT[0], 5):
    i = 0
    j = 1

elif np.around( LAT_DIFF[0], 5) == np.around( -1 * LAT[0], 5):
    i = -1
    j = -1

elif VAR=='irrigation_flux':
    i=0
    j=1

else:
    print("Latitude information of used masks doesn't match... :|")
    sys.exit( 2)
##ecearth data does not have units defined
if MODEL=='ecearth' or VAR=='albedo' or VAR=='TS_ebal' or VAR=='irrigation_flux' or VAR=='ts_ebal' or VAR=='wbgtid' or VAR=='wbgtid_iso_400W' or VAR=='SW_frac' or VAR=='LW_frac' or VAR=='avE' or VAR=='ts_ebal_x' or VAR=='ts_ebal_n' or VAR=='ts_ebal_meandaymax' or VAR=='ts_ebal_meandaymin' or VAR=='DTR' or VAR=='wbgtod' or VAR=='wbgtod_iso_400W':
	UNIT='/'
else:
	UNIT = f.variables[VAR].units
NLAT = len(LAT_DIFF)
NLON = len(LON_DIFF)
#DIFFERENCEMAP_ALLTIMESTEPS = f.variables[VAR][:,-1::-1,:] # time, lat, lon
if MODEL!='ecearth':
   DIFFERENCEMAP_ALLTIMESTEPS = f.variables[VAR][:,i::j,:] # time, lat, lon
   NSTEPS = len( DIFFERENCEMAP_ALLTIMESTEPS[:,0,0])
elif MODEL=='ecearth':
   DIFFERENCEMAP_ALLTIMESTEPS = f.variables[VAR][:,i::j] # time, rgrid
   NSTEPS = len( DIFFERENCEMAP_ALLTIMESTEPS[:,0])
   if DIFFERENCEMAP_ALLTIMESTEPS.ndim>2:                 #happens for variable with vertical layer (eg soil temperature) here we take top layer
       DIFFERENCEMAP_ALLTIMESTEPS=DIFFERENCEMAP_ALLTIMESTEPS[:,0,:]
#f.close()

# ----- CREATE CONSTANT INPUT MASKS

# 4 % EXTENDED NUMBER OF GRIDPOINTS IN LON DIRECTION NEEDED FOR INTERPOLATION AROUND 0 DEGREE W
# WILL BE RESTRICTED TO ORIGINAL GRID LATER.

# FOR MPI-ESM: INSTEAD OF 192 LONGITUDE ROWS WE ADD 8 LONGITUDE ROWS AND USE 200 LONGITUDE ROWS
#EXTEND_LON_NUMBER = int( np.around( len(LON) * 1.04, 0))
EXTEND_LON_NUMBER = len(LON) + 8

## CREATE EXTENDED CHESSBOARD MAP ON ALL LAND AND GLOBAL GRID POINTS -- non masked arrays
#CHESSBOARD_EXTENDED = np.tile( CHESSBOARD,2)[:,0:EXTEND_LON_NUMBER]
#GLOBAL_CHESSBOARD_EXTENDED = np.tile( GLOBAL_CHESSBOARD,2)[:,0:EXTEND_LON_NUMBER]

# CREATE EXTENDED CHESSBOARD MAP ON ALL LAND AND GLOBAL GRID POINTS -- masked arrays
if MODEL!='ecearth':  ##currently not done for ecearth as checkers are lkand only, only location where issue may arise is siberia alaskan border
   CHESSBOARD_EXTENDED = np.ma.masked_equal( np.tile( CHESSBOARD,2)[:,0:EXTEND_LON_NUMBER], value=FILL_VALUE)
   GLOBAL_CHESSBOARD_EXTENDED = np.ma.masked_equal( np.tile( GLOBAL_CHESSBOARD,2)[:,0:EXTEND_LON_NUMBER], value=FILL_VALUE)
   # CREATE EXTENDED SEA LAND MASK AND GLACIER MASK -- masked arrays
   LAND_EXTENDED = np.ma.masked_equal( np.tile( SEA_LAND_MASK,2)[:,0:EXTEND_LON_NUMBER], value=FILL_VALUE)
   GLACIER_EXTENDED = np.ma.masked_equal( np.tile( GLACIER, 2)[:,0:EXTEND_LON_NUMBER], value=FILL_VALUE)
   # REVERSE GLOBAL CHESSBOARD GRID POINTS (1-->0; 0-->1)
   GLOBAL_CHESSBOARD_EXTENDED_REVERSE = np.abs( GLOBAL_CHESSBOARD_EXTENDED - 1)
#else:
    ## TO DO add more than one value per lat band (normally should be 8) and need to add these at different spatial coordinates
  # index_list=[]
   #lat_list=[]
   #for i in range(len(LAT)):
   #    current_lat=LAT[i]
   #    if lat_list.count(current_lat)==0:
   #        lat_list.append(current_lat)
   #        index_list_lat= (LAT==current_lat).nonzero()
   #        min_lon=np.min(LON[index_list_lat])
   #        index_list_lon= (LON==min_lon).nonzero()
   #        j=0
   #        final_index=np.intersect1d(index_list_lat,index_list_lon)
   #        index_list.append(int(final_index[:]))
   #LAT_EXTENDED=np.zeros(len(LAT)+len(index_list))
   #LAT_EXTENDED[:len(LAT)]=LAT[:]
   #LAT_EXTENDED[len(LAT):]=LAT[index_list]
   #print(LAT_EXTENDED[len(LAT):])
   #LON_EXTENDED=np.zeros(len(LON)+len(index_list))
   #LON_EXTENDED[:len(LON)]=LON[:]
   #LON_EXTENDED[len(LON):]=LON[index_list]-1
   #print(LON_EXTENDED[len(LON):])
   #CHESSBOARD_EXTENDED=np.zeros(len(CHESSBOARD)+len(index_list))
   #CHESSBOARD_EXTENDED[:len(CHESSBOARD)]=CHESSBOARD[:]
   #CHESSBOARD_EXTENDED[len(CHESSBOARD):]=CHESSBOARD[index_list]
   #GLOBAL_CHESSBOARD_EXTENDED=np.zeros(len(GLOBAL_CHESSBOARD)+len(index_list))
   #GLOBAL_CHESSBOARD_EXTENDED[:len(GLOBAL_CHESSBOARD)]=GLOBAL_CHESSBOARD[:]
   #GLOBAL_CHESSBOARD_EXTENDED[len(GLOBAL_CHESSBOARD):]=GLOBAL_CHESSBOARD[index_list]

## CREATE EXTENDED SEA LAND MASK AND GLACIER MASK -- non masked arrays
#LAND_EXTENDED = np.tile( SEA_LAND_MASK,2)[:,0:EXTEND_LON_NUMBER]
#GLACIER_EXTENDED = np.tile( GLACIER,2)[:,0:EXTEND_LON_NUMBER]

def interpolate_nonlocal( chessboard_extended, differencemap_extended, land_extended, coastboxes):

    #---------coordinate of the grid boxes that are interpolated
    points = np.where( np.all( np.array(( chessboard_extended == 0, land_extended == 1)), axis = 0))
    values = differencemap_extended[np.all( np.array(( chessboard_extended == 0, land_extended == 1)), axis = 0)]
    #values = differencemap_extended[points]

    #grid_x, grid_y = np.mgrid[0:96:1, 0:land_extended.shape[1]:1]
    grid_x, grid_y = np.mgrid[0:land_extended.shape[0]:1, 0:land_extended.shape[1]:1]

    interped = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='linear')
    interped_nearest = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='nearest')

    non_local = cp( interped)
    non_local[coastboxes == 1] = interped_nearest[coastboxes == 1]
    # take values over the ocean directly
    non_local[land_extended == 0] = differencemap_extended[land_extended == 0]

    return non_local

def interpolate_total( chessboard_extended, differencemap_extended, land_extended, glacier_extended, coastboxes):

    #---------coordinate of the grid boxes that are interpolated
    points = np.where( chessboard_extended == 1)
    values = differencemap_extended[ chessboard_extended == 1]

    #grid_x, grid_y = np.mgrid[0:96:1, 0:land_extended.shape[1]:1]
    grid_x, grid_y = np.mgrid[0:land_extended.shape[0]:1, 0:land_extended.shape[1]:1]

    interped = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='linear')
    interped_nearest = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='nearest')

    total_timeseries_interpolated = cp( interped)
    # interpolate values at the coast differently
    total_timeseries_interpolated[coastboxes == 1] = interped_nearest[coastboxes == 1]
    # take values over the ocean directly
    total_timeseries_interpolated[land_extended == 0] = differencemap_extended[land_extended == 0]
    # take values over the glaciers directly --> this happens also implicitly for nonlocal interpolation
    total_timeseries_interpolated[glacier_extended == 1] = differencemap_extended[glacier_extended == 1]

    return total_timeseries_interpolated

def interpolate_local( chessboard_extended, local_extended, land_extended, glacier_extended, coastboxes):

    #---------coordinate of the grid boxes that are interpolated
    points = np.where( chessboard_extended == 1)
    values = local_extended[ chessboard_extended == 1]

    #grid_x, grid_y = np.mgrid[0:len(LAT):1, 0:land_extended.shape[1]:1]
    grid_x, grid_y = np.mgrid[0:land_extended.shape[0]:1, 0:land_extended.shape[1]:1]

    interped = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='linear')
    interped_nearest = interpolate.griddata( np.array(( points[1] * 1.0, points[0] * 1.0)).T, values,\
        (grid_y,grid_x), method='nearest')

    local_interp = cp( interped)
    local_interp[coastboxes == 1] = interped_nearest[coastboxes == 1]

    # MASK OUT WATER BODIES AND GLACIER TOGETHER
    local_interp[land_extended == 0] = 0
    local_interp[glacier_extended == 1] = 0

    return local_interp

def subtract(extended_diff_map, remote_extended):
    return extended_diff_map - remote_extended

def determine_coast_boxes(chessboard_extended, global_chessboard_extended, global_chessboard_extended_reverse):
    # coastpixels are boxes that are ocean or that are near the ocean,
    # such that an ocean grid box would be used for interpolating the lcc grid boxes
    coastboxes = np.zeros( chessboard_extended.shape)
    #coastboxes_2 = np.zeros( chessboard_extended.shape)

    for latindex in range( chessboard_extended.shape[0]):
        for lonindex in range(chessboard_extended.shape[1]):
            # in a 9 boxes window around the grid box, look if there are more global chessboard pixels
            # than land chessboard pixels
            # Johannes took global_chessboard_extended_reverse for his calculation....
            if (np.sum( global_chessboard_extended[latindex-2:latindex+3,lonindex-2:lonindex+3]) - np.sum( chessboard_extended[latindex-2:latindex+3,lonindex-2:lonindex+3]) > 0):
                coastboxes[latindex,lonindex]=1
            #if (np.sum( global_chessboard_extended_reverse[latindex-2:latindex+3,lonindex-2:lonindex+3]) -
                #np.sum( chessboard_extended[latindex-2:latindex+3,lonindex-2:lonindex+3]) > 0):
                #coastboxes_2[latindex,lonindex]=1
    return coastboxes#, coastboxes_2

def write_netcdf( total_timeseries, non_local_timeseries,local_timeseries, f, UNIT, LAT_DIFF, NSTEPS, NLAT, NLON, INTERPOLATE_METHOD):
#def write_netcdf( total_timeseries, non_local_timeseries, f, UNIT, LAT_DIFF, NSTEPS, NLAT, NLON):
    # USE COPY OF FILE TO CREATE NEW NETCDF FILE
    print("Writing netCDF4 file...")
    #ifile = FILE_DIFFERENCEMAP.replace(".nc","") + "_signal-separated.nc"
    #f = nc.Dataset( ifile, 'r+') # format='NETCDF4')

    # Define global attributes
    f.creation_date=str(dt.datetime.today())
    f.contact='felix.havermann@lmu.de; Ludwig-Maximilians-University Munich'
    f.comment='Produced with Script ' + os.path.basename(__file__)
    #f.title='Land cover fractions of MPIESM-1.2-LR LAMACLIMA ' + SIMULATION_TYPE + ' simulation'
    #f.subtitle=''
    #f.anything=''
    #f_restart.comment='Only the variable cover_fract_pot was changed to a 100 % forest world and all other cover types are set to zero.\n'\
    #f.comment='Only the variable cover_fract_pot was changed to a 100 % forest world and all other cover types are set to fract_small (1e-10).\n'\
        #'This file is used for a 100 % forest simulation within the LAMACLIMA project\n'\
        #'Produced with Script ' + os.path.basename(__file__)
    #f.createVariable(LCT_CONVERSION[key][1], np.float32,('lat','lon'), fill_value=FILL_VALUE)

    # NON-LOCAL SIGNAL
    f.createVariable( VAR + "_nonlocal", np.float64,('time','lat','lon'), fill_value=FILL_VALUE)
    #f.variables[VAR + "_nonlocal"][0,:,:] = non_local_timeseries[:,i::j,:]
    #f.variables[VAR + "_nonlocal"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
    f.variables[VAR + "_nonlocal"][:,:,:] = non_local_timeseries[:,i::j,:]
    f.variables[VAR + "_nonlocal"].longname = 'Non-local (=remote) effect of LCLM change'
    f.variables[VAR + "_nonlocal"].units = UNIT
    f.variables[VAR + "_nonlocal"].grid_type = 'gaussian'

    if INTERPOLATE_METHOD == "interpolate_local":
        # TOTAL SIGNAL
        total_timeseries_summed_up = local_timeseries[:,i::j,:] + non_local_timeseries[:,i::j,:]
        f.createVariable( VAR + "_total", np.float64,('time','lat','lon'), fill_value=FILL_VALUE)
        #f.variables[VAR + "_total"][0,:,:] = local_interpolated[:,i::j,:] + non_local_interpolated[:,i::j,:]
        f.variables[VAR + "_total"][:,:,:] = total_timeseries_summed_up
        #f.variables[VAR + "_total"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
        f.variables[VAR + "_total"].longname = 'Total (=local + non-local) effect of LCLM change'
        f.variables[VAR + "_total"].units = UNIT
        f.variables[VAR + "_total"].grid_type = 'gaussian'

        # LOCAL SIGNAL
        MASK = np.tile( CHESSBOARD[i::j,:], (NSTEPS,1,1))
        f.createVariable( VAR + "_local", np.float64,('time','lat','lon'), fill_value=FILL_VALUE)
        #f.variables[VAR + "_local"][0,:,:] = local_interped[:,i::j,:]
        #f.variables[VAR + "_local"][0,:,:] = local_timeseries[i::j,:]
        #f.variables[VAR + "_local"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
        f.variables[VAR + "_local"][:,:,:] = np.ma.masked_array( local_timeseries[:,i::j,:], MASK.mask)
        f.variables[VAR + "_local"].longname = 'Local effect of LCLM change'
        f.variables[VAR + "_local"].units = UNIT
        f.variables[VAR + "_local"].grid_type = 'gaussian'

    elif INTERPOLATE_METHOD == "interpolate_total":
        # TOTAL SIGNAL
        f.createVariable( VAR + "_total", np.float64,('time','lat','lon'), fill_value=FILL_VALUE)
        f.variables[VAR + "_total"][:,:,:] = total_timeseries[:,i::j,:]
        f.variables[VAR + "_total"].longname = 'Total effect of LCLM change'
        f.variables[VAR + "_total"].units = UNIT
        f.variables[VAR + "_total"].grid_type = 'gaussian'

        # LOCAL SIGNAL
        local_timeseries_difference = total_timeseries[:,i::j,:] - non_local_timeseries[:,i::j,:]

        GLACIER_MASK = np.tile( GLACIER[i::j,:], (NSTEPS,1,1))
        local_timeseries_difference[GLACIER_MASK==1] = 0

        MASK = np.tile( CHESSBOARD[i::j,:], (NSTEPS,1,1))
        #MASK = np.tile( CHESSBOARD, (NSTEPS,1,1))
        f.createVariable( VAR + "_local", np.float64,('time','lat','lon'), fill_value=FILL_VALUE)
        f.variables[VAR + "_local"][:,:,:] = np.ma.masked_array( local_timeseries_difference, MASK.mask)
        f.variables[VAR + "_local"].longname = 'Local effect of LCLM change'
        f.variables[VAR + "_local"].units = UNIT
        f.variables[VAR + "_local"].grid_type = 'gaussian'

    f.close()
    print("writing finished.")

def get_original_grid(grid_extended, longitude, time_step, timeseries):
    tmp1 = grid_extended[:, 4:len(longitude)]
    tmp2 = grid_extended[:, len(longitude):len(longitude)+4]
    timeseries[time_step,:,:] = np.hstack((tmp2, tmp1))

    return timeseries

#def get_original_grid_nonlocal(grid_extended, LON, step, non_local_timeseries):
    #tmp1 = grid_extended[:, 4:len(LON)]
    #tmp2 = grid_extended[:, len(LON):len(LON)+4]
    #non_local_timeseries[step,:,:] = np.hstack((tmp2, tmp1))

    #return non_local_timeseries

#def get_original_grid_local(grid_extended, LON, step, local_timeseries):
    #tmp1 = grid_extended[:, 4:len(LON)]
    #tmp2 = grid_extended[:, len(LON):len(LON)+4]
    #local_timeseries[step,:,:] = np.hstack((tmp2, tmp1))

    #return local_timeseries

# DETERMINE COASTBOXES TO APPLY NEAREST NEIGHBOUR CALCULATION FOR THOSE BOXES
if MODEL !='ecearth':
    coastboxes = determine_coast_boxes( CHESSBOARD_EXTENDED, GLOBAL_CHESSBOARD_EXTENDED, GLOBAL_CHESSBOARD_EXTENDED_REVERSE)

    non_local_timeseries = np.ma.masked_all((NSTEPS, NLAT, NLON))
    local_timeseries = np.ma.masked_all((NSTEPS, NLAT, NLON))
    total_timeseries = np.ma.masked_all((NSTEPS, NLAT, NLON))
else:   #can't use functions created above for ec-earth
    ##determine coastboxes
    coastboxes = np.zeros( LAT.shape)
    coastboxes_checkers = np.zeros( CHESSBOARD_LAT.shape)
    #coastboxes_2 = np.zeros( chessboard_extended.shape)
    #plt.scatter(LON,LAT,c=SEA_LAND_MASK,cmap='hot',s=0.1)
    #plt.colorbar()
    #plt.show()
    if os.path.isfile('/pf/b/b381334/signal_seperation/coast_checkers.sav'):
        coastboxes_checkers = pickle.load(open('/pf/b/b381334/signal_seperation/coast_checkers.sav', 'rb'))
        coastboxes = pickle.load(open('/pf/b/b381334/signal_seperation/coast.sav', 'rb'))
        translation_table = pickle.load(open('/pf/b/b381334/signal_seperation/trans_table.sav', 'rb'))
    else:
        from scipy.spatial import cKDTree
        import pickle
        tree = cKDTree(np.c_[LAT,LON])
        dd, ii = tree.query([0,0], k=8)
        translation_table=np.zeros((len(CHESSBOARD_LAT),1))  ##array to transfer from checkersboards land only grid to the full global grid
        translation_table_missing=np.zeros((len(CHESSBOARD_LAT),1))   ## array to check for values not matching between land only and global grids
        for i in range(len(CHESSBOARD_LAT)):
            if np.any(np.logical_and(np.round(LAT,1)==np.round(CHESSBOARD_LAT[i],1),np.round(LON,1)==np.round(CHESSBOARD_LON[i],1))) !=True:
                #print('shit')
                pass
                translation_table_missing[i,0]=1
            else:
            #print(np.logical_and(np.round(LAT,3)==np.round(CHESSBOARD_LAT[i],3),np.round(LON,3)==np.round(CHESSBOARD_LON[i],3)))
                #print(np.where(np.logical_and(np.round(LAT,3)==np.round(CHESSBOARD_LAT[i],3),np.round(LON,3)==np.round(CHESSBOARD_LON[i],3))))
                translation_table[i,0]=np.where(np.logical_and(np.round(LAT,2)==np.round(CHESSBOARD_LAT[i],2),np.round(LON,2)==np.round(CHESSBOARD_LON[i],2)))[0]
            ## find nearest neighbours of each location on land sea mask
                translation_table=translation_table.astype('int')
                dd, ii = tree.query(np.c_[LAT[translation_table[i,0]],LON[translation_table[i,0]]], k=8)
                # in a 9 boxes window around the grid box, look if there are more global chessboard pixels
                # than land chessboard pixels
                # Johannes took global_chessboard_extended_reverse for his calculation....
                #if (np.sum( SEA_LAND_MASK[0,ii]) > 1 and np.sum( SEA_LAND_MASK[0,ii])<7.5):
                if (np.sum( SEA_LAND_MASK[0,ii]) > 1 and np.sum( SEA_LAND_MASK[0,ii])<7):
                    #print(SEA_LAND_MASK[0,ii])
                    #print(ii)
                    coastboxes[translation_table[i,0]]=1
                    coastboxes_checkers[i]=1
                translation_table_missing[i,0]=np.nan

        #for i in range( len(LAT)):
            #if np.where(np.logical_and(np.round(CHESSBOARD_LAT,3)==np.round(LAT[i],3),np.round(CHESSBOARD_LON,3)==np.round(LON[i],3))):
                #translation_table[j,0]=i
                #print(j)
                #print(i)
                #print(translation_table[0:5,0])
                #time.sleep(0.5)

                ## find nearest neighbours of each location on land sea mask
                #dd, ii = tree.query(np.c_[LAT[i],LON[i]], k=8)
                # in a 9 boxes window around the grid box, look if there are more global chessboard pixels
                # than land chessboard pixels
                # Johannes took global_chessboard_extended_reverse for his calculation....
                #if (np.sum( SEA_LAND_MASK[0,ii]) > 1 and np.sum( SEA_LAND_MASK[0,ii])<7.5):
                    #print(SEA_LAND_MASK[0,ii])
                    #print(ii)
                    #coastboxes[i]=1
                    #coastboxes_checkers[j]=1

        pickle.dump(coastboxes, open('/pf/b/b381334/signal_seperation/coast.sav', 'wb'))
        pickle.dump(coastboxes_checkers, open('/pf/b/b381334/signal_seperation/coast_checkers.sav', 'wb'))
        pickle.dump(translation_table, open('/pf/b/b381334/signal_seperation/trans_table.sav', 'wb'))

##plot to check amount of missing matches, not sure why we have these but seems to be small enough not to cause any real issues, generally located at 0 long in boreal latitudes (both NH and SH)
#plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=translation_table_missing,cmap='hot',s=0.1)
#plt.colorbar()
#plt.show()

if MODEL != 'ecearth':
    for step in range( NSTEPS):
        #print("Step", step, "of")

        differencemap = DIFFERENCEMAP_ALLTIMESTEPS[step,:,:]
        ## CREATE EXTENDED DIFFERENCE MAPS FROM CTL, CROP, IRR, FRST, AND HARV EXPERIMENTS -- non masked arrays
        #DIFFERENCEMAP_EXTENDED = np.tile( differencemap,2)[:,0:EXTEND_LON_NUMBER]

        # CREATE EXTENDED DIFFERENCE MAPS FROM CTL, CROP, IRR, FRST, AND HARV EXPERIMENTS -- masked arrays
        DIFFERENCEMAP_EXTENDED = np.ma.masked_equal( np.tile( differencemap, 2)[:,0:EXTEND_LON_NUMBER], value=FILL_VALUE)

        # INTERPOLATE DIFFERENT EFFECTS AND RESTRICT TO ORIGINAL GRID
        #nonlocal_extended = interpolate_nonlocal( CHESSBOARD_EXTENDED, DIFFERENCEMAP_EXTENDED, LAND_EXTENDED, coastboxes)
        nonlocal_extended = delayed(interpolate_nonlocal)( CHESSBOARD_EXTENDED, DIFFERENCEMAP_EXTENDED, LAND_EXTENDED, coastboxes)

        #non_local_timeseries = delayed(get_original_grid_nonlocal)(nonlocal_extended, LON, step, non_local_timeseries)
        non_local_timeseries = delayed(get_original_grid)(nonlocal_extended, LON, step, non_local_timeseries)

        if INTERPOLATE_METHOD == "interpolate_total":
            # CALCULATE TOTAL EFFECTS BY JUST INTERPOLATING GRID POINTS WICH LCLM CHANGE
            total_extended = delayed(interpolate_total)( CHESSBOARD_EXTENDED, DIFFERENCEMAP_EXTENDED, LAND_EXTENDED, GLACIER_EXTENDED, coastboxes)

            total_timeseries = delayed(get_original_grid)(total_extended, LON, step, total_timeseries)

        elif INTERPOLATE_METHOD == "interpolate_local":
            #local_extended = DIFFERENCEMAP_EXTENDED - nonlocal_extended
            local_extended = delayed(subtract)( DIFFERENCEMAP_EXTENDED, nonlocal_extended)

            # INTERPOLATE LOCAL EFFECTS TO NO-LCC GRID BOXES
            #local_interped_extended = interpolate_local( CHESSBOARD_EXTENDED, local_extended, LAND_EXTENDED, GLACIER_EXTENDED, coastboxes)
            local_interped_extended = delayed(interpolate_local)( CHESSBOARD_EXTENDED, local_extended, LAND_EXTENDED, GLACIER_EXTENDED, coastboxes)

            local_timeseries = delayed(get_original_grid)(local_interped_extended, LON, step, local_timeseries)

        else:
            print("The interpolation method: <", INTERPOLATE_METHOD, "> could not be identified... :|")
            sys.exit( 2)

        #return local_interped, local, non_local

    total = delayed(write_netcdf)( total_timeseries, non_local_timeseries, local_timeseries, f, UNIT, LAT_DIFF, NSTEPS, NLAT, NLON, INTERPOLATE_METHOD)
    #total = delayed(write_netcdf)( total_timeseries, non_local_timeseries, f, UNIT, LAT_DIFF, NSTEPS, NLAT, NLON)
    total.compute()
else:##i.e. model==ecearth
    ##create global files here, in loop the signal seperation is performed for all land pixels only, after loop data is transferred from land grid to global grid
    non_local_timeseries=np.zeros([NSTEPS,len(CHESSBOARD)])
    total_timeseries=np.zeros([NSTEPS,len(CHESSBOARD)])
    local_timeseries=np.zeros([NSTEPS,len(CHESSBOARD)])

    print(NSTEPS)
    for step in range(NSTEPS):
        differencemap = cp(DIFFERENCEMAP_ALLTIMESTEPS[step,:])

        ##put interpolation in function as for other esms so we can use delayed function, also write 2 new write funcs one for reduced gaussion grid and one for gaussian
        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=CHESSBOARD,cmap='coolwarm',s=1,edgecolors='none',vmin=0,vmax=1)
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=coastboxes_checkers,cmap='coolwarm',s=1,edgecolors='none',vmin=0,vmax=1)
        #plt.colorbar()
        #plt.show()

        ##only retain values over land as we only have the checkerboard defined over land (note that this means we do not need to add extra lon at edge)
        diff_map=differencemap[translation_table[:,0]]
        #---------coordinate of the grid boxes that are interpolated
        points = np.where( np.all( np.array(( CHESSBOARD == 0)), axis = 0))
        points_x=CHESSBOARD_LAT[CHESSBOARD == 0]
        points_y=CHESSBOARD_LON[CHESSBOARD == 0]
        values = diff_map[CHESSBOARD == 0]   ##issue here since dfmap is global but should first be reduced to checkers grid
        #values = differencemap_extended[points]

        #grid_x, grid_y = np.mgrid[0:96:1, 0:land_extended.shape[1]:1]
        grid_x = CHESSBOARD_LAT
        grid_y = CHESSBOARD_LON
        interped = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='linear')
        interped_nearest = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='nearest')
        non_local = cp( interped)
        non_local[coastboxes_checkers == 1] = interped_nearest[coastboxes_checkers == 1]
        non_local_timeseries[step,:]=non_local
        # take values over the ocean directly
        #non_local[land_extended == 0] = differencemap_extended[land_extended == 0]

        #plt.scatter(CHESSBOARD_LON[coastboxes_checkers==1],CHESSBOARD_LAT[coastboxes_checkers==1],c=non_local[coastboxes_checkers==1],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=non_local_timeseries[step,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=non_local-non_local_timeseries[step,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

         #---------coordinate of the grid boxes that are interpolated
        points_x=CHESSBOARD_LAT[CHESSBOARD != 0]
        points_y=CHESSBOARD_LON[CHESSBOARD != 0]
        values = diff_map[CHESSBOARD != 0]

        interped = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='linear')
        interped_nearest = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='nearest')

        total_timeseries_interpolated = cp( interped)
        # interpolate values at the coast differently
        total_timeseries_interpolated[coastboxes_checkers == 1] = interped_nearest[coastboxes_checkers == 1]
        total_timeseries[step,:]=total_timeseries_interpolated

        local_diff_map=total_timeseries_interpolated-non_local
        values=local_diff_map[CHESSBOARD != 0]

        interped = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='linear')
        interped_nearest = interpolate.griddata( np.array(( points_y * 1.0, points_x * 1.0)).T, values,(grid_y,grid_x), method='nearest')

        local_timeseries_interpolated = cp( interped)
        # interpolate values at the coast differently
        local_timeseries_interpolated[coastboxes_checkers == 1] = interped_nearest[coastboxes_checkers == 1]
        local_timeseries[step,:]=local_timeseries_interpolated

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=diff_map,cmap='viridis',s=1,edgecolors='none')
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=local_timeseries[step,:],cmap='viridis',s=1,edgecolors='none')
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=non_local_timeseries[step,:]-total_timeseries[step,:],cmap='viridis',s=1,edgecolors='none')
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=non_local_timeseries[step,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

        #plt.scatter(CHESSBOARD_LON,CHESSBOARD_LAT,c=total_timeseries[step,:],cmap='viridis',s=1,edgecolors='none')
        #plt.colorbar()
        #plt.show()

        #add non local signal in raw output and overwrite only on land pixels while keeping original values over ocean for non-local and total signals
    full_local_timeseries=np.zeros([NSTEPS,len(LAT)])
    full_total_timeseries=cp(DIFFERENCEMAP_ALLTIMESTEPS)
    full_non_local_timeseries=cp(DIFFERENCEMAP_ALLTIMESTEPS)
    #test=DIFFERENCEMAP_ALLTIMESTEPS
    for t in range(len(CHESSBOARD_LAT)):
            full_non_local_timeseries[:,translation_table[t,0]]=non_local_timeseries[:,t]
            full_local_timeseries[:,translation_table[t,0]]=local_timeseries[:,t]
            full_total_timeseries[:,translation_table[t,0]]=total_timeseries[:,t]
    #test[:,translation_table[:,0]]=np.zeros([NSTEPS,len(CHESSBOARD_LAT)])
    #full_non_local_timeseries[:,translation_table[:,0]]=non_local_timeseries[:,:]
    #full_total_timeseries[:,translation_table[:,0]]=total_timeseries[:,:]
    #full_local_timeseries[:,translation_table[:,0]]=local_timeseries[:,:]

    #full_non_local_timeseries=np.zeros([NSTEPS,len(LAT)])
    #for i in range(len(LAT)):
    #    if i in translation_table[:,0]:
    #        full_non_local_timeseries[:,i]=non_local_timeseries[:,np.where(translation_table[:,0] == i)[0]][0]
            #full_total_timeseries[:,translation_table[j,0]]=total_timeseries[:,j]
            #full_local_timeseries[:,translation_table[j,0]]=local_timeseries[:,j]
    #    else:
    #        full_non_local_timeseries[:,i]=diff_map[:,i]

    #rint(non_local_timeseries[0,:])
    #print(full_non_local_timeseries[0,translation_table[:,0]])

    #plt.scatter(LON,LAT,c=full_local_timeseries[0,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #plt.scatter(LON,LAT,c=full_non_local_timeseries[0,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #plt.scatter(LON,LAT,c=full_total_timeseries[0,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #plt.scatter(LON,LAT,c=full_non_local_timeseries[0,:]-full_total_timeseries[0,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    print(full_total_timeseries.shape)
    ##here the code for thye write netcdf is given but adapted for the ecearth data reduced gaussian grid
    print("Writing netCDF4 file...")
    #ifile = FILE_DIFFERENCEMAP.replace(".nc","") + "_signal-separated.nc"
    #f = nc.Dataset( ifile, 'r+') # format='NETCDF4')
    # Define global attributes
    f.creation_date=str(dt.datetime.today())
    f.contact='felix.havermann@lmu.de; Ludwig-Maximilians-University Munich'
    f.comment='Produced with Script ' + os.path.basename(__file__)
    #f.title='Land cover fractions of MPIESM-1.2-LR LAMACLIMA ' + SIMULATION_TYPE + ' simulation'
    #f.subtitle=''
    #f.anything=''
    #f_restart.comment='Only the variable cover_fract_pot was changed to a 100 % forest world and all other cover types are set to zero.\n'\
    #f.comment='Only the variable cover_fract_pot was changed to a 100 % forest world and all other cover types are set to fract_small (1e-10).\n'\
        #'This file is used for a 100 % forest simulation within the LAMACLIMA project\n'\
        #'Produced with Script ' + os.path.basename(__file__)
    #f.createVariable(LCT_CONVERSION[key][1], np.float32,('lat','lon'), fill_value=FILL_VALUE)

        # NON-LOCAL SIGNAL
    f.createVariable( VAR + "_nonlocal", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
    #f.variables[VAR + "_nonlocal"][0,:,:] = non_local_timeseries[:,i::j,:]
    #f.variables[VAR + "_nonlocal"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
    f.variables[VAR + "_nonlocal"][:,:] = full_non_local_timeseries[:,:]
    f.variables[VAR + "_nonlocal"].longname = 'Non-local (=remote) effect of LCLM change'
    f.variables[VAR + "_nonlocal"].units = UNIT
    f.variables[VAR + "_nonlocal"].grid_type = 'gaussian'

    if INTERPOLATE_METHOD == "interpolate_local":
        # TOTAL SIGNAL
        full_local_timeseries[np.isnan(full_local_timeseries)] = 0
        total_timeseries_summed_up = full_local_timeseries[:,:] + full_non_local_timeseries[:,:]
        f.createVariable( VAR + "_total", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
        #f.variables[VAR + "_total"][0,:,:] = local_timeseries[:,:] + non_local_timeseries[:,:]
        f.variables[VAR + "_total"][:,:] = total_timeseries_summed_up
        #f.variables[VAR + "_total"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
        f.variables[VAR + "_total"].longname = 'Total (=local + non-local) effect of LCLM change'
        f.variables[VAR + "_total"].units = UNIT
        f.variables[VAR + "_total"].grid_type = 'gaussian'

        # LOCAL SIGNAL
        #MASK = np.tile( CHESSBOARD[i::j,:], (NSTEPS,1,1))
        f.createVariable( VAR + "_local", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
        #f.variables[VAR + "_local"][0,:,:] = local_interped[:,i::j,:]
        #f.variables[VAR + "_local"][0,:,:] = local_timeseries[i::j,:]
        #f.variables[VAR + "_local"][1:,:,:] = np.zeros(( NSTEPS-1, NLAT, NLON))
        f.variables[VAR + "_local"][:,:] =  full_local_timeseries[:,:]
        f.variables[VAR + "_local"].longname = 'Local effect of LCLM change'
        f.variables[VAR + "_local"].units = UNIT
        f.variables[VAR + "_local"].grid_type = 'gaussian'

    elif INTERPOLATE_METHOD == "interpolate_total":
        # TOTAL SIGNAL
        f.createVariable( VAR + "_total", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
        f.variables[VAR + "_total"][:,:] = full_total_timeseries[:,:]
        f.variables[VAR + "_total"].longname = 'Total effect of LCLM change'
        f.variables[VAR + "_total"].units = UNIT
        f.variables[VAR + "_total"].grid_type = 'gaussian'

        # LOCAL SIGNAL
        full_local_timeseries_difference = full_total_timeseries[:,:] - full_non_local_timeseries[:,:]

        #MASK = np.tile( CHESSBOARD[:], (NSTEPS,1,1))
        #MASK = np.tile( CHESSBOARD, (NSTEPS,1,1))
        f.createVariable( VAR + "_local", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
        f.variables[VAR + "_local"][:,:] = full_local_timeseries_difference[:,:]
        f.variables[VAR + "_local"].longname = 'Local effect of LCLM change'
        f.variables[VAR + "_local"].units = UNIT
        f.variables[VAR + "_local"].grid_type = 'gaussian'

        # LOCAL SIGNAL
        #MASK = np.tile( CHESSBOARD[:], (NSTEPS,1,1))
        #MASK = np.tile( CHESSBOARD, (NSTEPS,1,1))
        f.createVariable( VAR + "_real_local", np.float64,('time','rgrid'), fill_value=FILL_VALUE)
        f.variables[VAR + "_real_local"][:,:] = full_local_timeseries[:,:]
        f.variables[VAR + "_real_local"].longname = 'Local effect of LCLM change'
        f.variables[VAR + "_real_local"].units = UNIT
        f.variables[VAR + "_real_local"].grid_type = 'gaussian'

    #plt.scatter(LON,LAT,c=full_non_local_timeseries[4,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #plt.scatter(LON,LAT,c=full_total_timeseries[4,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #plt.scatter(LON,LAT,c=full_local_timeseries[4,:],cmap='coolwarm',s=1,edgecolors='none',vmin=-2,vmax=2)
    #plt.colorbar()
    #plt.show()

    #f.close()
    print("writing finished.")
    #create new zeros array with globval coverage, use translation table and loop and fill values up, where land sea mask ==0 just fill non local and total up with diffmap vals

        #final results achieved still to merge and write to netcdf
    #if False:

        #fig = plt.figure()
        #plt.imshow(non_local, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(local, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(local_interped, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(local_interped_masked, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(SEA_LAND_MASK, cmap="Wistia",interpolation= 'None')
        ##cb = plt.colorbar(im, orientation='horizontal')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(LAND_EXTENDED, cmap="Wistia",interpolation= 'None')
        ##cb = plt.colorbar(im, orientation='horizontal')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(CHESSBOARD_EXTENDED, cmap="Wistia",interpolation= 'None')
        ##cb = plt.colorbar(im, orientation='horizontal')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(GLOBAL_CHESSBOARD_EXTENDED, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(GLOBAL_CHESSBOARD_EXTENDED_REVERSE, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(coastboxes, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

        #fig = plt.figure()
        #plt.imshow(coastboxes_2, cmap="Wistia", interpolation= 'None')
        #plt.colorbar()
        #plt.show()

    #if False:
        #fig = plt.figure()
        #plt.subplot(2,2,1)
        #plt.imshow(local_interped, interpolation= 'None')#,vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,2)
        #plt.imshow(local_extended, interpolation= 'None')#,vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,3)
        #plt.imshow(local_interped_extended, interpolation= 'None')#,vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,4)
        #plt.imshow(CHESSBOARD_EXTENDED, interpolation= 'None')#,vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

    #if False:
        #fig = plt.figure()
        #plt.subplot(2,2,1)
        #plt.imshow(local_interped, interpolation= 'None',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,2)
        #plt.imshow(local, interpolation= 'None',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,3)
        #plt.imshow(non_local, interpolation= 'None',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.subplot(2,2,4)
        #plt.imshow(interped_interp1, interpolation= 'None',vmin=-2,vmax=2)
        #plt.colorbar()
        #plt.show()

    #m = Basemap(projection='cyl', lon_0 = 0, resolution='c')
    ##meshlon, meshlat = m(LON, LAT)

    #fig = plt.figure()
    ##m.drawcoastlines(linewidth=0.5)
    ##m.drawparallels(np.arange(-20,61,20),labels=[0,1,0,0],fontsize=20, color='grey')

    ##m.drawparallels(np.arange(-80,81,40))#, labels=[0,1,0,0])
    ##m.drawmeridians(np.arange(-160,160,20), labels=[0,0,0,1])#,fontsize=8,linewidth=0,labelstyle="+/-")

    ##m.pcolormesh( LON, LAT, local_interped + non_local, latlon=True, cmap=cmap_bwr_jwi,\
        ##norm=norm,vmin=-plotmax, vmax=plotmax, rasterized=True, zorder=1)

    ##im = m.pcolormesh( LON, LAT, local_interped, latlon=True, cmap=cmap_bwr_jwi,\
        ##norm=norm,vmin=-plotmax, vmax=plotmax, rasterized=True, zorder=1)

    #im = m.pcolormesh(LON, LAT, CHESSBOARD, cmap='Wistia', latlon=True)# rasterized=True, zorder=1
    #im = m.pcolormesh(LON, LAT, differencemap, cmap='Wistia', latlon=True)# rasterized=True, zorder=1
    #im = m.pcolormesh(LON, LAT, non_local, cmap='Wistia', latlon=True)#, zorder=1)#, rasterized=True,
    #im = m.pcolormesh(LON, LAT, local, cmap='Wistia', latlon=True)#, zorder=1)#, rasterized=True,
    #im = m.pcolormesh(LON, LAT, local_interped, cmap='Wistia', latlon=True)#, rasterized=True, zorder=1)

    #plt.show()
    #im = m.pcolormesh(ii - 0.05, jj - 0.05, interpolated_grid, cmap='bwr', vmin= -5, vmax= 5)
    #cb = plt.colorbar(im, orientation='horizontal')
    #plt.show()


import os
import sys
import shlex
from benedict import BeneDict

def is_in_path(file, folder):
    """
    Disallow symlinks
    https://stackoverflow.com/questions/3812849/how-to-check-whether-a-directory-is-a-sub-directory-of-another-directory
    """
    folder = os.path.join(os.path.realpath(folder), '')
    file = os.path.realpath(file)  # expand symlinks to real path

    # return true, if the common prefix of both is equal to directory
    # e.g. /a/b/c/d.rst and directory is /a/b, the common prefix is /a/b
    return os.path.commonprefix([file, folder]) == folder

# docker run -ti  -v /sailhome/jimfan/vision:/home/jimfan/vision:rw $(URL) bash
class Dockerly:
    def __init__(self, config_file='~/.dockerly.yml'):
        try:
            self.config = BeneDict.load_yaml_file(os.path.expanduser(config_file))
        except FileNotFoundError:
            print('must specify a config file ' + config_file)
            raise
        for key in ['container_root', 'host_root', 'default_image']:
            assert key in self.config, 'config "{}" missing'.format(key)
        for k, v in self.config.items():
            if v == '_fill_yours_':
                raise ValueError('please fill in key "{}" in {}'.format(k, config_file))
        self.container_root = self.config.container_root
        assert os.path.isabs(self.container_root), \
            'container_root must be an absolute path: ' + self.container_root
        # must use realpath, otherwise relative path will be wrong
        self.host_root = os.path.realpath(os.path.expanduser(self.config.host_root))
        self.ports = self.config.ports
        self.default_image = self.config.default_image
        self._docker_exe = 'nvidia-docker' if self.config.nvidia else 'docker'
        self._dry_run = False

    def _run_system(self, cmd):
        if self._dry_run:
            print('>>', cmd)
        else:
            os.system(cmd)

    def _mount_flag(self, host_root, container_root, read_only=False):
        host_root = os.path.expanduser(host_root)
        if read_only:
            read_option = 'ro'
        else:
            read_option = 'rw'
        return '-v {}:{}:{}'.format(host_root, container_root, read_option)

    def _ports_flag(self):
        flag = ''
        for port_spec in self.ports:
            flag += '-p {} '.format(port_spec)
        return flag

    def dry_run(self, dry_run):
        self._dry_run = dry_run
        return self

    def run(self, cmd, *other_options):
        """
        other_options: flags to pass to docker run
        """
        cmd = '{docker} run -ti --privileged {others} {mount} {ports} {image} {cmd}'.format(
            docker=self._docker_exe,  # nvidia-docker
            others=' '.join(other_options),
            ports=self._ports_flag(),
            mount=self._mount_flag(self.host_root, self.container_root),
            image=self.default_image,
            cmd=shlex.quote(cmd),
        )
        # replace excessive whitespace for better dry-run:
        cmd = ' '.join(cmd.split())
        self._run_system(cmd)

    def run_from_cwd(self, cmd):
        """
        Change relative cwd ("current working dir") in the docker container
        """
        host_cwd = os.getcwd()
        assert is_in_path(host_cwd, self.host_root), \
            'you must run this command within subpaths of `host_root` (in ~/.dockerly.yml) because it will be projected into relative path within the container.'
        host_relpath = os.path.relpath(host_cwd, self.host_root)
        container_abspath = os.path.join(self.container_root, host_relpath)
        self.run(cmd, '--workdir="{}"'.format(container_abspath))


'''
@file: retrieve_social_media_data.py
@author: John3Kim
@desc: Make an OOP-based implementation to collect social media
from different sources such as Reddit and Twitter.

TODO: Simplify the get_submissions and get_comments methods
'''

from abc import ABC, abstractmethod
from typing import List, Dict
import datetime
from datetime import date, timedelta
import time
import json
import requests
import logging
from export_to_json import build_error_text_file

logging.basicConfig(level=logging.DEBUG,
                    format=' %(asctime)s - %(levelname)s - %(message)s')

class CollectRedditDataAbstract(ABC):
    '''
    Abstract class for Reddit data to show what methods we need to incorporate
    '''

    def __init__(self) -> None:
        pass

    @abstractmethod
    def get_submissions() -> None:
        raise NotImplementedError("Please implement the get_submissions method.")

    @abstractmethod
    def get_comments_id_from_submissions_id(self) -> None:
        raise NotImplementedError("Please implement the get_comments_id_from_submissions_id method.")

    @abstractmethod
    def get_comments(self) -> None:
        raise NotImplementedError("Please implement the get_comments method.")

class CollectRedditPSIOData(CollectRedditDataAbstract):
    '''
    Methods needed to collect Reddit data

    We need two sub-classes:
    one for pushshift.io -> Historical data
    one for praw -> recent/real-time data
    This class involves pushshift.io only

    Attributes __init__ :

        self.time_delay: int -> Time delay that is used for each RESTful API request (in seconds). Default is 2 seconds.
        self.number_results: int -> The number of results that can be retrieved from each request.  Ranges from 25 to 500.
                                    Default is 500 but a request may not retrieve 500 results.
    '''
    def __init__(self) -> None:
        self.time_delay = 2
        self.number_results = 500

    def get_submissions(self, query:str, subreddit:str, before_date:str, after_date:str, num_retries:int = 1) -> json:
        '''
        Get a set of reddit submissions. This represents a single call of the posts endpoint.

        Arguments:
            query: str -> The search query that you want to run. You can write a single term
                or combine with the following boolean commands:
                AND, OR, (), NOT

                Search terms may be stemmed. e.g. dogs resolves as dog for search.

            subreddit:str -> The subreddit, or categories devoted to one topic, that you want to search into.
            start_date: datetime.time -> The start date of the query.
            end_date: datetime.time -> The end date of the query.
            num_retries:int -> In the event that a request fails, we rerun the submissions query.  Default is 1 retry.

        Returns:
            json: A JSON object that contains the results of a single request using the submissions endpoint.
        '''
        for retry in range(num_retries + 1):

            try:
                submissions_api_call = f'https://api.pushshift.io/reddit/search/submission/?q={query}&subreddit={subreddit}&after={before_date}&before={after_date}&limit={self.number_results}'
                submissions_requests = requests.get(submissions_api_call)
                time.sleep(self.time_delay)

            except json.decoder.JSONDecodeError:
                logging.info(f"Failed to parse: {submissions_api_call}")
                logging.info(f"Export error file.")
                build_error_text_file(submissions_api_call,"submissions_error")
                logging.info(f"Retrying request.")
                continue

            except requests.HTTPError as http_error:
                logging.info(f"HTTP error: {http_error}")
                logging.info(f"Retrying request.")
                continue

            # If we're successful in obtaining our data, no need to retry
            break

        return submissions_requests.json()

    def get_comments_id_from_submissions_id(self, submission_id:str, num_retries:int = 1) -> List[Dict[str,str]]:
        '''
        Given a single submissions id (called id), represented as base36, we obtain the comments id.

        This is dependent on obtaining ids from the posts

        Arguments:
            submission_id:str -> The submisson id that is retrieved from pusushift.io's submissions endpoint.
            num_retries:int -> In the event that a request fails, we rerun the submissions query.  Default is 1 retry.

        Returns:
            List[Dict[str,str]] -> A list of dictionaries where each element of the list will contain the original submission
                                   id and its corresponding comments id.  Multiple elements means that there are multiple comments
                                   per submission id.

        '''

        list_comments_submission_id = []

        for retry in range(num_retries + 1):

            try:
                submission_id_api_call = f"https://api.pushshift.io/reddit/submission/comment_ids/{submission_id}"
                sub_id_to_comment_id_requests = requests.get(submission_id_api_call)
                time.sleep(self.time_delay)

                sub_id_to_comment_id_requests = sub_id_to_comment_id_requests.json()["data"]
                # turn the list of ids into a json into an object

                for comment_id in sub_id_to_comment_id_requests:
                    list_comments_submission_id.append({"submission_id":submission_id,"comment_id": comment_id})

            except json.decoder.JSONDecodeError:
                logging.info(f"Failed to parse: {submission_id_api_call}")
                logging.info(f"Export error file.")
                build_error_text_file(submission_id_api_call,"submission_id_error")
                logging.info(f"Retrying request.")
                continue

            except requests.HTTPError as http_error:
                logging.info(f"HTTP error: {http_error}")
                logging.info(f"Retrying request.")
                continue

            # If we're successful in obtaining our data, no need to retry
            break

        if len(list_comments_submission_id) == 0:
            return [{"submission_id":submission_id, "comment_id": "N/A"}]

        return list_comments_submission_id

    def get_comments(self, list_comments_ids: List[str], num_retries:int = 1) -> json:

        '''
        Gets the comments of a post, based on the comments ids.

        Arguments:
            list_comments_ids: List[str] -> Each element will contain a comments id, represented as a base-36 string.
            num_retries:int -> In the event that a request fails, we rerun the submissions query.  Default is 1 retry.
        '''

        strings_comments = ','.join(list_comments_ids)

        for retry in range(num_retries + 1):

            try:
                comments_api_call = f'https://api.pushshift.io/reddit/search/comment?ids={strings_comments}'
                comments_requests = requests.get(comments_api_call)
                time.sleep(self.time_delay)

            except json.decoder.JSONDecodeError:
                logging.info(f"Failed to parse: {comments_api_call}")
                logging.info(f"Export error file.")
                build_error_text_file(comments_api_call,"comment_error")
                logging.info(f"Retrying request.")
                continue

            except requests.HTTPError as http_error:
                logging.info(f"HTTP error: {http_error}")
                logging.info(f"Retrying request.")
                continue

            # If we're successful in obtaining our data, no need to retry
            break

        return comments_requests.json()


import numpy as np
import cv2

def calculate (objp, imgp, grayp):
    print("calculating calibration result...")
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objp, imgp, grayp.shape[::-1], None, None)

    # Compute mean of reprojection error
    mean_error = 0
    for i in range(len(objpoints)):
        imgpoints2, _ = cv2.projectPoints(objpoints[i], rvecs[i], tvecs[i], mtx, dist)
        error = cv2.norm(imgpoints[i], imgpoints2, cv2.NORM_L2) / len(imgpoints2)
        mean_error += error

    with open('calib_result.txt', 'a') as the_file:
        the_file.write('Camera Matrix: \n')
        the_file.write(str(np.concatenate(mtx, axis=0).tolist()) + '\n')
        the_file.write('Distortion Coefficients: \n')
        the_file.write(str(np.concatenate(dist, axis=0).tolist()) + '\n')
        the_file.write('Rotation Vector: \n')
        the_file.write(str(np.concatenate(rvecs, axis=0).tolist()) + '\n')
        the_file.write('Translation Vector: \n')
        the_file.write(str(np.concatenate(tvecs, axis=0).tolist()) + '\n')
        the_file.write("Mean reprojection error:\n" + str(mean_error / len(objpoints)) + '\n')
        the_file.write('--------------------------------')

# Global variables
cap = cv2.VideoCapture(0)
global_count = 0
font = cv2.FONT_HERSHEY_SIMPLEX

# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 75, 0.001)

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((6*9, 3), np.float32)
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.

while True:
    # Capture frame-by-frame
    ret, frame = cap.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    if global_count % 30 != 0:
        # Global count to skip frames
        global_count += 1
        continue

    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, (9, 6), None)

    # If found, add object points, image points (after refining them)
    if ret:
        objpoints.append(objp)
        corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners2)
        # Draw and display the corners
        frame = cv2.drawChessboardCorners(frame, (9, 6), corners2, ret)

    cv2.imshow('img', frame)
    key = cv2.waitKey(1) & 0xFF

    # if the `q` key was pressed, break from the loop
    if key == ord("q"):
        cv2.destroyAllWindows()
        calculate(objpoints, imgpoints, gray)


''' Let S(n) = Σa+b+c over all triples (a,b,c) such that: a, b, and c are prime
numbers; a < b < c < n; a+1, b+1, and c+1 form a geometric sequence. Find
S(10**8) '''

from collections import defaultdict
from pyprimesieve import primes as primes_to

def main():
    ''' Driver function '''
    end = 10**4
    print(progression(end))

def progression(end):
    ''' Build undirected graph where nodes are prime numbers and edges are
    the ratio between the larger prime and the smaller prime (plus 1 in both
    cases). Return the sum of nodes that are part of a geometric progression
    '''
    total = 0
    primes = primes_to(end)
    graph = defaultdict(dict)
    for i, p1 in enumerate(primes[:-1]):
        if graph[p1].keys():
            min_r = min(graph[p1].keys())
            max_r = max(graph[p1].keys())
        else:
            min_r = 1
            max_r = ((end-1) / (p1+1))**0.5
        for p2 in primes[i+1:]:
            r = (p2 + 1) / (p1 + 1)
            if (p2+1)*r > end and max_r < r or min_r > r:
                break
            if r in graph[p1]:
                total += p1 + p2 + graph[p1][r]
            else:
                graph[p1][r] = p2
            if r in graph[p2]:
                total += p1 + p2 + graph[p2][r]
            else:
                graph[p2][r] = p1
    return total

if __name__ == "__main__":
    main()


from _fd_method import *

if __name__=='__main__':
    (S0, K, r, q, T, sigma, omega) = (50, 60, 0.03, 0.01, 1, 0.4, -1)
    (S, K, r, q, T, sigma, option_type) = (50, 60, 0.03, 0.01, 1, 0.4, 'put')

    (Smin, Smax, Ns, Nt) = (0, 4*np.maximum(S,K), 100, 100)

    # explicit method should have small time steps and large stock price
    option = FullyExplicitEu(S, K, r, q, T, sigma, option_type, Smin, Smax, Ns=20, Nt=100)
    print(option.price())
    # implicit method always converge
    option2 = FullyImplicitEu(S, K, r, q, T, sigma, option_type, Smin, Smax, Ns, Nt)
    print(option2.price())

    # so as the Crank Nicolson
    option3 = CrankNicolsonEu(S, K, r, q, T, sigma, option_type, Smin, Smax, 100, 100)
    print(option3.price())

    print(blackscholes(S0, K, r, q, T, sigma, omega))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 13:21:00 2020

@author: rajat

Credit goes to David Tingley and Adrien Peyrache:
    https://github.com/DavidTingley/RnR_methods
"""
import sys
import fastica
import numpy as np
import scipy.stats as scst
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

# function to get reactivation strength
def ReactStrength(Qref, Qtar, method='pca'):
    # Qref = nCells x nTimes
    nCells = Qref.shape[0]
    if nCells != Qref.shape[0]:
        sys.error(2)

    # calculate correlation matrix
    Cref = np.corrcoef(Qref)

    # run pca
    pca = PCA()
    pca = pca.fit(Cref)
    PCs = pca.components_ # used for projection matrix
    lambdas = pca.explained_variance_ratio_*nCells

    # Marcenko-Pastur threshold
    lMax = (1 + np.sqrt(nCells/Qref.shape[1]))**2
    nPCs = np.sum(lambdas>lMax)
    phi = lambdas[:nPCs]/lMax

    if method=='ica':
        _, PCs, _ = fastica.fastica(Qref, nCells, algorithm='parallel')
        PCs = PCs/np.array([np.sqrt(np.sum(PCs**2,1))]*nCells)

    # get reactivation strength for each template across time bins
    QtarZ = scst.zscore(Qtar,1).T
    scoreTar = QtarZ.dot(PCs[:,:nPCs])

    # this trick is used to get rid of the diagonal term in react. strength
    tmp = (QtarZ**2).dot(PCs[:,:nPCs]**2)
    R = scoreTar**2 - tmp

    return R, phi

"""
Qref = np.load('Qrun.npy', allow_pickle=True)
Qtar = np.load('Qsleep.npy', allow_pickle=True)
Qref = np.delete(Qref, (80,83), axis=0)
Qtar = np.delete(Qtar, (80,83), axis=0)

R1, _ = ReactStrength(Qref, Qtar, method='pca')
R2, _ = ReactStrength(Qref, Qtar, method='ica')

plt.plot(R1[:,0])
plt.plot(R2[:,0])
"""


op_list = ['ld', 'cp', 'st', 'set', 'nop', 'alu', 'alui', 'mvm', 'vvo', 'hlt', 'jmp', 'beq', 'alu_int', 'crs']
aluop_list = ['add', 'sub', 'sna', 'mul', 'sigmoid'] # sna is also used by mvm isntruction
op_list_tile = ['send', 'receive', 'halt']

latency = {"ld": None,
           "cp": 1,
           "st": None,
           "set": 1,
           "nop": 1,
           "alu": 1,
           "alui": 1,
           "mvm": 1000,
           "vvo": 1,
           "hlt": 1,
           "jmp": 1,
           "beq": 1,
           "alu_int": 1,
           "crs": 1,
           "send": None,
           "receive": None,
           "halt": 1
          }