# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 17:46:19 2021

@author: Ben
"""

import pandas as pd
import seaborn as sns
from scipy.stats import pearsonr
from scipy.stats import spearmanr
import matplotlib.pyplot as plt

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from numpy import cov
from sklearn.preprocessing import StandardScaler

from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn import metrics
from sklearn.metrics import accuracy_score
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
students_df = pd.read_csv('with_grades_df_2020_2021.csv')
students_df.rename( columns={'Unnamed: 0':'anonymous_id'}, inplace=True )
#%%##### Grade Distribution
def plot_grades(df):
    grades = df['OVERALL_GRADE'].value_counts()
    grades.plot.bar()
plot_grades(students_df)

course_views = students_df['Study Area: [Module 2020/2021] CM4107 - Full Time: Advanced Artificial Intelligence'];
gradeDictionary = {5:'A',4:'B',3:'C',2:'D',1:'E',0:'F'}

#%%###### TARGET LABEL ENCODER
l1 = LabelEncoder()

#fit will transform labels into a numeric value
l1.fit(students_df['OVERALL_GRADE'])

#transform the existing student_df dataset
students_df.OVERALL_GRADE = l1.transform(students_df.OVERALL_GRADE);
######

## intuitive transform
## reverse the label encode values
## previously 0=A, transform 0=NS, 1=E etc.
for idx, val in enumerate(students_df['OVERALL_GRADE']):
    if(val == 6):
        #print("grade is F")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",5,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 0
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 5):
        #print("grade is NS")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",5,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 0
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 4):
        #print("grade is E")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",4,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 1
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 3):
        #print("grade is D")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",3,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 2
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 2):
        #print("grade is C")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",2,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 3
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 1):
        #print("grade is B")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",1,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 4
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
    elif(val == 0):
        #print("grade is A")
        #print(students_df.loc[idx]['OVERALL_GRADE'],"\n")
        #print("grade:",val,"\n")
        #print("current student df:",val,"\n")
        #print("current grade label:",0,"\n")
        students_df.at[idx, 'OVERALL_GRADE'] = 5
        #print("new label:", students_df.loc[idx]['OVERALL_GRADE'],"\n")

#%%###### Course Views Logistic Regression
course_grades = pd.concat([course_views,students_df['OVERALL_GRADE']],axis='columns')
course_grades = course_grades.rename(columns={'Study Area: [Module 2020/2021] CM4107 - Full Time: Advanced Artificial Intelligence':'course_views'})

covariance = cov(course_grades['course_views'],course_grades['OVERALL_GRADE'])
print(covariance)
#positive covariance shows there is a linear relationship

#pearson shows there is a moderate linear relationship
course_corr, _ = pearsonr(course_views, course_grades['OVERALL_GRADE'])
print("pearson course views linear relation with grade", course_corr)

##Two variables may be related by a nonlinear relationship, such that the relationship is stronger or weaker across the distribution of the variables.
course_views_spearman, _ = spearmanr(course_grades['course_views'],course_grades['OVERALL_GRADE'])
print('Spearmans correlation: %.3f' % course_views_spearman)

#%% Find the corelation for every feature
def get_correlations(students_df):
    features = students_df.drop(['anonymous_id','COURSEWORK_1','COURSEWORK_2','OVERALL_GRADE'], axis=1)
    #for each feature in students_df
    corelations = list()

    for feature in features:
        feature_extracted = students_df[feature]
        current_grade = students_df['OVERALL_GRADE']
        corr, _ = pearsonr(feature_extracted,current_grade)
        corelations.append([feature,corr])

    corelations = pd.DataFrame(corelations)
    corelations.columns = ['Feature', 'Corelation']

    corelations = corelations.sort_values('Corelation', ascending=False)

    return corelations

corelations = get_correlations(students_df)
#print(corellations.head(20))
#%% Get the columns from correlations and select features from student df
topActivities = corelations.head(20)
topList = list()

for val in topActivities['Feature']:
    topList.append(val)

topList.append('OVERALL_GRADE')
features = students_df[topList]
#print(len(features.index))
#%% Use the Random Forest classifier
X = features.iloc[:,0:20].values
y = features.iloc[:, 20].values

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)

sc = StandardScaler()

X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)

classifier = RandomForestClassifier(n_estimators=20)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)

print("Random Forest Accuracy: ",accuracy_score(y_test, y_pred))

#%% SVM
clf = svm.SVC(kernel='linear')
clf.fit(X_train, y_train)

y_pred = clf.predict(X_test)

print("SVM Accuracy:",metrics.accuracy_score(y_test, y_pred))

#%%
X = features.iloc[:,0:len(features.columns)-1].values
y = features.iloc[:, len(features.columns)-1].values
def rf_predict(X,y):
    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)
    #print(skf.get_n_splits(X, y))

    param_grid = {
                     'n_estimators': [5, 10, 15, 20, 100],
                     'max_depth': [2, 5, 7, 9, 10, 11]
                 }

    sc = StandardScaler()
    X_train = sc.fit_transform(X_train)
    X_test = sc.fit_transform(X_test)

    random_forest_data=list()
    scores = ['precision', 'recall']

    rfc = RandomForestClassifier(n_jobs=-1,n_estimators=50)
    cv = KFold(n_splits=10, shuffle=True, random_state=42)
    rfc_model = GridSearchCV(estimator=rfc, param_grid=param_grid, cv=cv)
    rfc_model.fit(X_train, y_train)
    print(rfc_model.best_score_)
    print(rfc_model.best_params_)

    rfc_score = rfc_model.score(X_test, y_test)
    print('Test Accuracy: %.3f' % rfc_score)
    return rfc_score

rf_predict(X,y)

def svm_predict(X,y):
    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)

    param_grid = [
      {'C': [1, 10, 100, 1000], 'kernel': ['linear']},
      {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},
     ]

    sc = StandardScaler()
    X_train = sc.fit_transform(X_train)
    X_test = sc.fit_transform(X_test)
    svc = svm.SVC(kernel='linear')
    cv = KFold(n_splits=10, shuffle=True, random_state=42)
    svc_model = GridSearchCV(estimator=svc, param_grid=param_grid, cv=cv)
    svc_model.fit(X_train, y_train)
    print(svc_model.best_score_)
    print(svc_model.best_params_)

    svm_score = svc_model.score(X_test, y_test)
    print('Test Accuracy: %.3f' % svm_score)

    return svm_score

svm_predict(X,y)

#%%Testing
testFeatures = [10,15,20,25,30,35,40,45,50,55,60,65,70,75,len(students_df.columns)-4]

def best_scores_with_feature_sets(testFeatures):
    all_svms=list();
    all_rfs=list();
    best_df = pd.DataFrame({'Number of Features':testFeatures})
    best_df["Random Forest"] = np.nan
    best_df["Support Vector Classifier"] = np.nan

    micro_df = pd.DataFrame({'Number of Features':testFeatures})
    micro_df["Random Forest"] = np.nan
    micro_df["Support Vector Classifier"] = np.nan

    print(best_df)
    for value in testFeatures:
        topActivities = corelations.head(value)
        topList = list()

        for val in topActivities['Feature']:
            topList.append(val)

        topList.append('OVERALL_GRADE')
        features = students_df[topList]

        X = features.iloc[:,0:value].values
        y = features.iloc[:, value].values

        rf_score = rf_predict(X, y)
        svc_score = svm_predict(X, y)
        #rf_score = rf_predict(X, y)
        #svc_score = svm_predict(X, y)
        #all_svms.append([value,rf_score])
        #all_rfs.append([value,svc_score])

        best_df.loc[best_df['Number of Features'] == value, 'Random Forest'] = rf_score
        best_df.loc[best_df['Number of Features'] == value, 'Support Vector Classifier'] = svc_score

    best_df = best_df.melt('Number of Features', var_name='ML Algorithm on 2020/2021',  value_name='Accuracy')
    sns.set_style("darkgrid")
    sns.factorplot(x="Number of Features", y="Accuracy", hue='ML Algorithm on 2020/2021', data=best_df, palette=sns.color_palette('summer', n_colors=2))
    plt.show()

    return best_df

best_dfs = best_scores_with_feature_sets(testFeatures)


'''
页面;登录页面
'''
from appium.webdriver.webdriver import WebDriver
import unittest
from selenium.webdriver.support.wait import WebDriverWait
from src.common import Base_page
from appium.webdriver.common import mobileby
import random
import time

class xiatui (Base_page.Base_page,unittest.TestCase):
    by = mobileby.MobileBy ()

    # ------------------------采购收货单页面------------------------
    # 采购收货单-订单列表-已审核订单（通过xpath进行定位）
    cgshd_shenhe_dingdan_liebiao_sj = (by.XPATH, "//*[@class='android.view.View'][@content-desc='已审核'][@resource-id='labelapaudit']")

    # 封装操作（点击操作）
    def click_cgshd_shenhe_dingdan_liebiao_sj(self):
        # 判断能否进行点击操作，如果能够点击操作，赋值1，同时输出点击成功的提示
        try:
            self.find_element (*self.cgshd_shenhe_dingdan_liebiao_sj).click()
            self.dy_cgshd_shenhe_dingdan_liebiao_sj = 1
            print ("订单列表-已审核订单，点击成功")
        # 如果不能进行点击操作，赋值2，同时输出点击不成功的提示，并且输出当前页的截图
        except:
            self.dy_cgshd_shenhe_dingdan_liebiao_sj = 2
            print ("订单列表-已审核订单，输出当前截图")
            self.driver.get_screenshot_as_file ("C:\\Users\\Administrator\\PycharmProjects\\pythonProject\\cqydtest\\src\\image\\click_cgshd_shenhe_dingdan_liebiao_sj.png")

    # 采购收货单-订单列表-已审核订单-下推（通过xpath进行定位）
    cgshd_shenhe_dingdan_xiatui = (by.XPATH, "//*[@class='android.view.View'][@content-desc='下推']")

    # 封装操作（点击操作）
    def click_cgshd_shenhe_dingdan_xiatui(self):
        # 判断能否进行点击操作，如果能够点击操作，赋值1，同时输出点击成功的提示
        try:
            self.find_element (*self.cgshd_shenhe_dingdan_xiatui).click ()
            self.dy_cgshd_shenhe_dingdan_xiatui = 1
            print ("下推成功")
        # 如果不能进行点击操作，赋值2，同时输出点击不成功的提示，并且输出当前页的截图
        except:
            self.dy_cgshd_shenhe_dingdan_xiatui = 2
            print ("下推失败，输出当前截图")
            self.driver.get_screenshot_as_file ("C:\\Users\\Administrator\\PycharmProjects\\pythonProject\\cqydtest\\src\\image\\click_cgshd_shenhe_dingdan_xiatui.png")


from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q, Count
from django.core.cache import cache
from rest_framework.views import APIView
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import permissions, generics

from .models import Book
from user.models import User
from .serializers import AuthorSerializer, BookSerializer, BookDownloadSerializer
from .payment import make_payment

class ListBooks(generics.ListAPIView):
    queryset = Book.objects.all()
    serializer_class = BookSerializer

    def list(self, request):
        search_str = request.GET.get('search', None)
        author_surname = request.GET.get('authorsurname', None)
        if search_str:
            q_name = Q()
            q_surname = Q()
            q_title = Q()
            search_str = search_str.strip().split(' ')
            for el in search_str:
                q_name |= Q(authors__name__icontains=el)
                q_surname |= Q(authors__surname__icontains=el)
                q_title |= Q(title__icontains=el)
            queryset = Book.objects.filter(q_name | q_surname | q_title).distinct()
        elif author_surname:
            queryset = Book.objects.filter(authors__surname=author_surname)
        else:
            queryset = self.get_queryset()
        serializer = BookSerializer(queryset, many=True, context={'request': request})
        return Response(serializer.data)

class BookDetails(generics.RetrieveAPIView):
    serializer_class = BookDownloadSerializer

    def get(self, request, pk):
        queryset = Book.objects.get(pk=pk)
        serializer = BookDownloadSerializer(queryset, context={'request': request})
        return Response(serializer.data)

@api_view(['GET'])
def buy_book(request, pk):
    if request.method == 'GET' and request.user.is_authenticated:
        try:
            user = User.objects.get(pk=request.user.id)
            book = Book.objects.get(pk=pk)
            result = make_payment(user.cards, book.price)
            if result:
                user.books.add(book)
                return Response({"message": "OK!"})
            else:
                return Response({"message": "payment error"}, status=status.HTTP_402_PAYMENT_REQUIRED)
        except ObjectDoesNotExist:
            return Response({"message": "object not found"}, status=status.HTTP_404_NOT_FOUND)
    else:
        return Response({"message": "error"}, status=status.HTTP_403_FORBIDDEN)

class ListMyBooks(generics.ListAPIView):
    queryset = Book.objects.all()
    serializer_class = BookSerializer
    permission_classes = (permissions.IsAuthenticated,)

    def list(self, request):
        queryset = self.queryset.filter(users__id=request.user.id)
        serializer = BookSerializer(queryset, many=True, context={'request': request})
        return Response(serializer.data)

class MainView(generics.ListAPIView):
    serializer_class = BookSerializer

    def get_permissions(self):
        if self.request.method in permissions.SAFE_METHODS:
            return (permissions.AllowAny(),)

    def list(self, request):
        stored_top = cache.get('books_top')
        if stored_top:
            #print('from cache')
            serializer = BookSerializer(stored_top, many=True, context={'request': request})
            return Response(serializer.data)
        else:
            #print('from db')
            queryset = Book.objects.annotate(users_count=Count('users')).order_by('-users_count')[:3]
            serializer = BookSerializer(queryset, many=True, context={'request': request})
            return Response(serializer.data)


#!/usr/bin/env python

"""
Plot lipid-lipid g(r) results.
"""

import scipy
import scipy.integrate
import re

class Same:
	"""Make sure items are the same between replicates."""
	def __init__(self): self.this = []
	def add(self,item): self.this.append(item)
	def get(self):
		if not all([self.this[0]==i for i in self.this[1:]]): raise Exception('not same')
		else: return self.this[0]

def make_legend(ax,keys=None):
	# legend at last tile
	legendspec = []
	for sn_general,sns in replicate_mapping:
		if not keys or sn_general in keys:
			legendspec.append(dict(name=extra_labels[sn_general],
				patch=mpl.patches.Rectangle((0,0),1.0,1.0,fc=color_by_simulation(sns[0]))))
	patches,labels = [list(j) for j in zip(*[(i['patch'],i['name']) for i in legendspec])]
	legend = ax.legend(patches,labels,loc='upper left',bbox_to_anchor=(1.0,0.0,1.,1.))
	frame = legend.get_frame()
	frame.set_edgecolor('k')
	frame.set_facecolor('w')
	return legend

def reducer(pairings,pair_residues):
	"""Final reduction in the data."""
	post = dict([])
	for pnum,(pairname,pair) in enumerate(zip(pairings,pair_residues)):
		post[pairname] = {}
		for sn_group,sns_this in replicate_mapping:
			obs,nmols,total_areas = [],Same(),[]
			for sn in sns_this:
				mn = actinlink_monolayer_indexer(sn,abstractor='lipid_chol_com')
				dat = data[sn]['data']
				key = 'counts_mn%d_%s:%s'%(mn,pairname[0],pairname[1])
				if key not in dat:
					status('reducing %s, %s SKIP'%(sn,key))
					continue
				else: status('reducing %s, %s'%(sn,key))
				obs.append(dat[key])
				nmols.add([np.in1d(dat['resnames'][dat['monolayer_indices']==mn],p).sum()
					for p in pair])
				total_areas.append(dat['total_area'])
			if len(obs) not in [0,2]: raise Exception('something has gone wrong')
			elif len(obs)==0: continue
			# average replicates
			counts = np.array(obs).mean(axis=0)
			nmols = nmols.get()
			total_area = np.mean(total_areas) #! total area is averaged between simulations for norming
			density = nmols[0]*nmols[1]/total_area
			post[pairname][sn_group] = dict(counts=counts,total_area=total_area,density=density)

@autoload(plotrun)
def load():
	data,calc = plotload('lipid_rdfs')
	sns = work.sns()
	# generics
	mode = ['summary','cdf'][-1]
	xmax_cumulative = 2.0
	cutoff,binsize = [calc['lipid_rdfs']['calcs']['specs'][i] for i in ['cutoff','binsize']]
	scanrange = np.arange(0,cutoff,binsize)
	middles = (scanrange[1:]+scanrange[:-1])/2
	areas = np.array([np.pi*(binsize)*middles[i]*2 for i in range(len(middles))])
	# collect all possible pairings
	pairings = list(set([tuple(i) for j in [
		data[sn]['data']['pairnames_mn%d'%actinlink_monolayer_indexer(sn,abstractor='lipid_chol_com')]
		for sn in sns] for i in j]))
	pairname_to_pair = dict([(tuple(i),j) for i,j in zip(*[data[sn]['data']['%s_mn%d'%(p,actinlink_monolayer_indexer(sn,abstractor='lipid_chol_com'))] for p in ['pairnames','pairs']])])
	pair_residues = [pairname_to_pair[p] for p in pairings]
	color_by_simulation = actinlink_color_by_simulation
	replicate_mapping = actinlink_replicate_mapping
	extra_labels = actinlink_extra_labels
	#! it would be nice to have this calculation in a separate function but then it will not have globals.
	#! ... this issue needs documenting along with the load magic
	post = dict([])
	for pnum,(pairname,pair) in enumerate(zip(pairings,pair_residues)):
		post[pairname] = {}
		for sn_group,sns_this in replicate_mapping:
			obs,nmols,total_areas = [],Same(),[]
			for sn in sns_this:
				mn = actinlink_monolayer_indexer(sn,abstractor='lipid_chol_com')
				dat = data[sn]['data']
				key = 'counts_mn%d_%s:%s'%(mn,pairname[0],pairname[1])
				if key not in dat:
					status('reducing %s, %s SKIP'%(sn,key))
					continue
				else: status('reducing %s, %s'%(sn,key))
				obs.append(dat[key])
				nmols.add([np.in1d(dat['resnames'][dat['monolayer_indices']==mn],p).sum()
					for p in pair])
				total_areas.append(dat['total_area'])
			if len(obs) not in [0,2]: raise Exception('something has gone wrong')
			elif len(obs)==0: continue
			# average replicates
			counts = np.array(obs).mean(axis=0)
			nmols = nmols.get()
			total_area = np.mean(total_areas) #! total area is averaged between simulations for norming
			density = nmols[0]*nmols[1]/total_area
			post[pairname][sn_group] = dict(counts=counts,total_area=total_area,density=density,nmols=nmols)

def plot_radial_distributions(figname,mode='summary',**kwargs):
	"""Plot RDFs or CDFs of the RDFs."""
	# plot lipid-lipid RDFs on one panel
	reps_subset = kwargs.get('groups',None)
	if reps_subset: replicate_mapping_this = [(i,j) for i,j in replicate_mapping if i in reps_subset]
	else: replicate_mapping_this = replicate_mapping
	axes,fig = square_tiles(len(pairings),figsize=(12,12),hspace=0.4,wspace=0.4)
	for pnum,(pairname,pair) in enumerate(zip(pairings,pair_residues)):
		for sn_group,sns_this in replicate_mapping_this:
			if sn_group not in post[pairname]: continue
			counts,nmols,total_area,density = [post[pairname][sn_group][k]
				for k in ['counts','nmols','total_area','density']]
			ax = axes[pnum]
			#! show 90% of the maximum range
			xmax = 0.9*np.sqrt(total_area)
			valid = middles<xmax
			if mode=='cdf':
				valid = middles<xmax_cumulative
				vals = np.array([scipy.integrate.simps(counts[:i]/areas[:i]/density)
					for i in range(2,len(counts))])[valid]
				ax.plot(middles[valid],vals[valid],color=color_by_simulation(sns_this[0]),lw=2)
				ax.set_xlim((0,xmax_cumulative))
				ax.set_ylabel('$\int{g(r)}$')
			elif mode=='summary':
				valid = middles<xmax
				ax.plot(middles[valid],(counts/areas/density)[valid],
					color=color_by_simulation(sns_this[0]),lw=2)
				ax.axhline(1.0,lw=1,c='k')
				ax.set_xlim((0,xmax))
				ax.set_ylabel('$g(r)$')
				ax.axhline(1.0,lw=0.5,c='k')
			else: raise Exception
			ax.set_xlim((0,xmax_cumulative))
			ax.set_title('%s-%s'%tuple([work.vars['names']['short'].get(p,p) for p in pairname]))
			ax.tick_params(axis='y',which='both',left='off',right='off',labelleft='on')
			ax.tick_params(axis='x',which='both',top='off',bottom='off',labelbottom='on')
			ax.set_xlabel('$r\,(nm)$')
	legend = make_legend(axes[-1],keys=[i for i,j in replicate_mapping_this])
	picturesave('fig.lipid_rdfs.%s'%figname,directory=work.plotdir,meta={},extras=[legend])

@autoplot(plotrun)
def plot_distributions():
	figspec = {
		'summary':{'mode':'summary'},
		'summary.20':{'mode':'summary','groups':['pip2_20_no_chol','pip2_20']},
		'cumulative':{'mode':'cdf'},}
	for figname,spec in figspec.items(): plot_radial_distributions(figname=figname,**spec)

@autoplot(plotrun)
def plot_cumulative_compare():
	figname = 'cumulative_compare'
	axes,fig = square_tiles(len(pairings),figsize=(12,12),hspace=0.4,wspace=0.4)
	heights = {}
	for pnum,(pairname,pair) in enumerate(zip(pairings,pair_residues)):
		ax = axes[pnum]
		heights[pairname] = {}
		for snum,(sn_group,sns_this) in enumerate(replicate_mapping):
			if sn_group not in post[pairname]: continue
			counts,nmols,total_area,density = [post[pairname][sn_group][k]
				for k in ['counts','nmols','total_area','density']]
			# mimic the cdf routine
			valid = middles<xmax_cumulative
			vals = np.array([scipy.integrate.simps(counts[:i]/areas[:i]/density)
				for i in range(2,len(counts))])[valid]
			# the last value occurs at the xmax_cumulative
			#! check that this is precise?
			heights[pairname][sn_group] = vals[-1]
			ax.bar(snum,vals[-1],color=color_by_simulation(sns_this[0]),width=1.0)
		ax.tick_params(axis='y',which='both',left='off',right='off',labelleft='on')
		ax.tick_params(axis='x',which='both',top='off',bottom='off',labelbottom='on')
		ax.set_xticks([])
		ax.set_title('%s-%s'%tuple([work.vars['names']['short'].get(p,p) for p in pairname]))
	ymin = min([min(i.values()) for i in heights.values()])
	ymax = max([max(i.values()) for i in heights.values()])
	for ax in axes:
		ax.set_ylim((0.9*ymin,1.1*ymax))
	legend = make_legend(axes[-1])
	picturesave('fig.lipid_rdfs.%s'%figname,directory=work.plotdir,meta={},extras=[legend])

plotrun.routine = None
if __name__=='__main__': pass


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#   .-.                              _____                                  __
#   /v\    L   I   N   U   X       / ____/__   ___   ___   ____ ___   ___  / /_  __  __
#  // \\                          / / __/ _ \/ __ \/ __ `/ ___/ __ `/ __ \/ __ \/ / / /
# /(   )\                        / /_/ /  __/ /_/ / /_/ / /  / /_/ / /_/ / / / / /_/ /
#  ^^-^^                         \____/\___/\____/\__, /_/   \__,_/ .___/_/ /_/\__, /
#                                                /____/          /_/          /____/

"""
This script draws flow maps for rendering in a GIS. It does this by drawing an
interpolation between origins and destinations in the output coordinate system,
combined with an across-track point. This module depends on GDAL/OGR, pyproj,
shapely, and scipy, each of which is freely available and open-source.

You must supply the script with a csv file where each row represents an arc
with a flow magnitude. Specific information about the required format of that
csv file is in the README file acompanying this script.

This is written for Python 3; it may not work on Python 2.

Written against versions (via the Anaconda Python distribution):
Python 3.5.3
scipy 0.19.0
gdal 2.1.0
shapely 1.5.16
pyproj 1.9.5.1

Please feel free to contact the author, Dr. Paulo Raposo, at
pauloj.raposo@outlook.com. Thanks for your interest!

- Paulo, http://volweb.utk.edu/~praposo

"""

dependencies = """Python 3, scipy, gdal, shapely, pyproj (Proj.4)"""

progName = "Interpolated Flow Maps"
__version__ = "0.3, March 2018"

license = """
# Under MIT License:
#
# Copyright (c) 2018 Paulo Raposo, Ph.D. - pauloj.raposo@outlook.com
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""

# Notes /////////////////////////////////////////////////////////////////////////////

# TODO: make script write any other attributes to output, too.
# TODO: allow user to skew curves.

# Imports ///////////////////////////////////////////////////////////////////////////

try:
    import pyproj
    from pyproj import Proj
except ImportError:
    print("""This script depends on the pyproj library, which isn't installed in
    this Pyhon environment. Please install the library or use an environment
    with it installed. Exiting.""")
    exit()
try:
    import osgeo
    from osgeo import ogr, gdal, osr
    gdal.UseExceptions()
except ImportError:
    print("""This script depends on the GDAL library, which isn't installed in this
    Python environment. Please install the library or use a Python environment
    with GDAL installed. Exiting.""")
    exit()
try:
    from scipy.interpolate import CubicSpline, Akima1DInterpolator, PchipInterpolator
except ImportError:
    print("""This script depends on the scipy library, version 0.18 or greater,
    which isn't installed in this Python environment. Please install the
    library or use a Python environment with scipy installed. Exiting.""")
    exit()
try:
    import shapely.affinity as aff
    import shapely.geometry
    from shapely.geometry import Point
except ImportError:
    print("""This script depends on the shapely library, version 1.5.17.post1 or
    greater, which isn't installed in this Python environment. Please install
    the library or use a Python environment with scipy installed. Exiting.""")
    exit()

import os, csv, argparse, math, re
from urllib import request
import numpy as np
# import datetime, logging

# Functions & Classes ///////////////////////////////////////////////////////////////////

def calcOrthogonalVector(aVector, clockwise):
    """Given a 2-vector (i.e., a numpy 2D array), this returns the orthogonal
    vector of the same magnitude in either the clockwise or counter-clockwise
    direction, corresponding to the given boolean flag."""
    if clockwise == True:
        return np.array([aVector[1], aVector[0] * -1.0])
    else:
        return np.array([aVector[1] * -1.0, aVector[0]])

def calcAlongSegmentCoords(xy1, xy2, asf):
    """Given the endpoints of a line segment, and an 'along-segment fraction,'
    returns the coordinates of the midpoint by calculating simple x and y
    ranges, each times the value of asf."""
    ydiff = (float(xy2[1]) - float(xy1[1]))
    xdiff = (float(xy2[0]) - float(xy1[0]))
    yMid = float(xy1[1]) + (asf * ydiff)
    xMid = float(xy1[0]) + (asf * xdiff)
    return (xMid, yMid)

def generateInterpolator(xSeries, ySeries, aType):
    """Given an x and y series (assumed to be in sync with each other!),
    and a string indicating which type of interpolant is asked for,
    returns the calculated interpolation function from the appropriate
    SciPy method. Options are 'cs' for cubic spline, 'a' for Akima, and
    'pchip' for PCHIP."""
    interpo = acceptedInterpolators[aType]
    if interpo == CubicSpline:
        # Boundary Conditions are crucial to shape here!!!!
        # TODO: study these and do something smart here using first and second derivatives at line ends.
        # see https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.interpolate.CubicSpline.html
        i = interpo(xSeries, ySeries, bc_type=((2, 0.0), (2, 0.0))) # A 'natural' spline, with second derivatives at ends = 0.0.
    else:
        # Akima or PCHIP
        i =  interpo(xSeries, ySeries)
    return i

def createLineString(xyList):
    """Creates an ORG LineString geometry, given a sequenced list of
    vertices, being tuples of format (x,y)."""
    line = ogr.Geometry(ogr.wkbLineString)
    for v in xyList:
        line.AddPoint(v[0], v[1]) # x then y
    return line

def createAField(dstLayer, fieldName, fieldType):
    """Simple field maker for this script."""
    new_field = ogr.FieldDefn(fieldName, fieldType)
    dstLayer.CreateField(new_field)

def strictly_increasing(L):
    """Checks strict increasing monotonicity of a list of numbers.
    Returns True or False."""
    # With thanks to "6502": http://stackoverflow.com/questions/4983258/python-how-to-check-list-monotonicity
    return all(x<y for x, y in zip(L, L[1:]))

def gdal_error_handler(err_class, err_num, err_msg):
    # https://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html#install-gdal-ogr-error-handler
    # https://trac.osgeo.org/gdal/wiki/PythonGotchas#Gotchasthatarebydesign...orperhistory
    errtype = {
            gdal.CE_None:'None',
            gdal.CE_Debug:'Debug',
            gdal.CE_Warning:'Warning',
            gdal.CE_Failure:'Failure',
            gdal.CE_Fatal:'Fatal'
    }
    err_msg = err_msg.replace('\n',' ')
    err_class = errtype.get(err_class, 'None')
    print('Error Number: %s' % (err_num))
    print('Error Type: %s' % (err_class))
    print('Error Message: %s' % (err_msg))

def filterProj4String(p4string):
    """
    Removes the '+units' flag and value from a Proj4 string, and the
    '+ellps' flag and value if there is a '+datum' flag and value,
    since those seems to trip pyproj up. Seems kludgy. Argh.
    """

    def should_keep_flag(flag, should_remove_ellps):
        """
        Determines if a flag should be kept
        in the string.
        """
        if flag.startswith("+units="):
            return False

        if should_remove_ellps and flag.startswith("+ellps="):
            return False

        return True

    has_datum_flag = "+datum=" in p4string

    flags = [
        flag for flag in p4string.split(" ")
        # Having a datum means specifying an ellipse is redundant.
        if should_keep_flag(flag, has_datum_flag)
    ]

    # Return the final filtered Proj4 string
    outstring = " ".join(flags)
    print("String returned: {}".format(outstring))
    return outstring

class LicenseAction(argparse.Action):
    """A custom action to get the license to print, and then exit."""
    def __init__(self, nargs=0, **kw):
        super().__init__(nargs=nargs, **kw)
    def __call__(self, parser, namespace, values, option_string=None):
        print(license)

# The acceptable values for interpolator types as code and SciPy method pairs.
acceptedInterpolators = {
    "cs":    CubicSpline,
    "a":     Akima1DInterpolator,
    "pchip": PchipInterpolator
}

# The acceptable output file types by file extension and their driver names for OGR.
# These are a hand-picked subset of these: http://gdal.org/1.11/ogr/ogr_formats.html.
# These are chosen mainly because they're available in OGR by default (i.e., OGR is
# compiled supporting them by default), and they carry attribute fields over well.
typesAndDrivers = {
    ".shp":     "ESRI Shapefile",
    ".geojson": "GeoJSON",
    ".kml":     "KML",
    ".gml":     "GML",
    ".gmt":     "GMT"
}

# Script /////////////////////////////////////////////////////////////////////////////////

def main():

    """The main method of this script, for making flow maps, hot and fresh
     to your table (or desk), all flowy and map-like - that's amore!"""

    # Constants, defaults, etc. /////////////////////////////////////////////////////////

    # Set up error handler for GDAL
    gdal.PushErrorHandler(gdal_error_handler)

    # EPSG:4326 WGS 84 - for required input.
    wgs84RefURL = "http://spatialreference.org/ref/epsg/4326/" # Retrieved string below on 2017-06-01
    epsgWGS84Proj4 = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
    wgs84SR = osr.SpatialReference()
    wgs84SR.ImportFromProj4(epsgWGS84Proj4)

    # EPSG:3785 Web Mercator - for default output.
    webMercatorRefURL = "http://spatialreference.org/ref/epsg/3785/" # Retrieved string below on 2017-06-01
    epsgWebMercProj4 = "+proj=merc +lon_0=0 +k=1 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +towgs84=0,0,0,0,0,0,0 +no_defs" # manually removed +units=m
    wmSR = osr.SpatialReference()
    wmSR.ImportFromProj4(epsgWebMercProj4)

    # Output field names
    textFieldNames = ["Orig", "Dest"]
    floatFieldNames = ["FlowMag", "OrigLat", "OrigLon", "DestLat", "DestLon"]

    # Various default values
    outP4 = epsgWebMercProj4
    interpolator = "cs"
    alongSegmentFraction = 0.5
    devFraction = 0.15
    vertsPerArc = 300
    clockWise = True
    verbose = False
    # gr = 0.25 / 1.618 # For the Golden Ratio, phi.

    # Usage messages, and parse command line arguments.
    descString = progName + " -- " + "A script for making flow maps in GIS, using interpolated paths, by Paulo Raposo (pauloj.raposo@outlook.com).\nUnder MIT license. \nWritten for Python 3 - may not work on 2. Dependencies include: " + dependencies + "."
    parser = argparse.ArgumentParser(prog = progName, description = descString, formatter_class = argparse.RawDescriptionHelpFormatter)
    parser.add_argument("ROUTES", help = "CSV file specifying routes and magnitudes. Coordinates must be lat and lon in WGS84. Please see the README file for required formatting.")
    parser.add_argument("OUTPUTFILE", help = "File path and name for output shapefile. The containing directory must already exist. The file format is determined from the extension given here, with these options: .shp, .kml, .gml, .gmt, or .geojson.")
    parser.add_argument("--outproj4", help = "Output projected coordinate system to draw flow arcs in, given as a Proj.4 string. Often available at spatialreference.org. Three input formats are acceptable: a Proj.4 string, a URL starting with 'http://' to the Proj.4 string for a coodinate system on spatialreference.org (e.g., http://spatialreference.org/ref/esri/53012/proj4/), or a full path to a plain text file containing (only) a Proj.4 string. Default output projection is Web Mercator (" + webMercatorRefURL + ").")
    parser.add_argument("-i", "--interpolator", help = "The type of interpolator to use. Options are 'cs' for cubic spline (the default), 'a' for Akima, and 'pchp' for PCHIP.")
    parser.add_argument("-a", "--asf", help = "The 'along-segment fraction' of the straight line segment between start and end points of a flow at which an orthogonal vector will be found to construct the deviation point. Expressed as a number between 0.0 and 1.0. Default is 0.5.")
    parser.add_argument("-d", "--dev", help = "The across-track distance at which a deviated point should be established from the straight-line vector between origin and destination points, expressed as a fraction of the straight line distance. Larger values make arcs more curved, while zero makes straight lines. Negative values result in right-handed curves. Default is 0.15.")
    parser.add_argument("-v", "--vpa", help = "The number of vertices the mapped arcs should each have. Must be greater than 3, but typically should be at least several dozen to a few hundred or so. Default is " + str(vertsPerArc) + ".")
    parser.add_argument("--ccw", default = False, action = "store_true",  help = "Sets the across-track deviation point on the left by rotating the across-track vector counter-clockwise. Changes the directions that arcs curve in. Default is clockwise.")
    parser.add_argument("--verbose", default = False, action = "store_true", help = "Be verbose while running, printing lots of status messages.")
    parser.add_argument("--version", action = "version", version = "%(prog)s " + __version__)
    parser.add_argument("--license", action = LicenseAction, nargs = 0, help = "Print the script's license and exit.")
    #
    args = parser.parse_args()

    # Set variables, do various checks on input arguments.
    pathAndFile, ext = os.path.splitext(args.OUTPUTFILE)
    try:
        ogrDriverName = typesAndDrivers[ext.lower()]
    except:
        print("Output file must be of one of these types: {}. Exiting.".format(str(list(typesAndDrivers.keys()))))
        exit()
    if args.vpa:
        vertsPerArc = args.vpa
    if args.outproj4:
        if args.outproj4.startswith("http://"):
            # URL.
            f = request.urlopen(args.outproj4)
            outP4 = filterProj4String( str(f.read(), "utf-8") ) # Decode from byte string.
        elif os.path.exists(args.outproj4):
            # Assuming a path to a text file has been passed in.
            f = open(args.outproj4)
            outP4 = filterProj4String( f.read() )
            f.close()
        else:
            # Proj.4 string.
            outP4 = filterProj4String( args.outproj4 )
    if args.interpolator:
        if args.interpolator in acceptedInterpolators.keys():
            interpolator = args.interpolator
        else:
            print("Didn't understand the specified interpolator type. Acceptable codes are {}. Exiting.".format(str(list(acceptedInterpolators.keys()))))
            exit()
    if args.asf:
        alongSegmentFraction = float(args.asf)
        if alongSegmentFraction <= 0.0 or alongSegmentFraction >= 1.0:
            print("Along-segment fraction {} is out of bounds, must be within 0.0 and 1.0. Exiting.".format(str(alongSegmentFraction)))
            exit()
    if args.dev:
        devFraction = float(args.dev)
    if args.ccw:
        clockWise = False
    if args.verbose:
        verbose = True

    # Build the necessary coordinate systems.
    pIn = Proj(epsgWGS84Proj4)
    try:
        pOut = Proj(outP4)
    except:
        print("Unable to define projection from input provided for Proj4. Please ensure the string is valid. Exiting.")
        exit()
    outSR = osr.SpatialReference()
    outSR.ImportFromProj4(outP4)

    # Create an output file where the user specified, and add attribute fields to it.
    if verbose:
        print("Preparing file for output...")
    driver = ogr.GetDriverByName(ogrDriverName)
    outFile = args.OUTPUTFILE
    dst_ds = driver.CreateDataSource(outFile)
    fName = os.path.splitext(os.path.split(outFile)[1])[0]
    dst_layer = dst_ds.CreateLayer(fName, outSR, geom_type = ogr.wkbLineString)
    layer_defn = dst_layer.GetLayerDefn()
    for field in textFieldNames:
        createAField(dst_layer, field, ogr.OFTString)
    for field in floatFieldNames:
        createAField(dst_layer, field, ogr.OFTReal)

    # Open and read the CSV.
    # Each row is an arc/route in the flow map. Process each row into a feature.
    if verbose:
        print("Reading csv...")
    with open(args.ROUTES) as csvfile:
        dReader = csv.DictReader(csvfile, delimiter = ',', quotechar = '"')
        # Reference fields by their headers; first row taken for headers by default.
        # Find every unique origin point, and separate arcs into groups by origin point,
        # stored in a dictionary.
        originGroups = {} # Entries return lists of lists.
        originKeys = []

        for row in dReader: # Populate originGroups.

            # These strings are the headers (and fields) the input csv must have.
            oName  =  row["OrigName"]
            oLat   =  row["OrigLat"]
            oLon   =  row["OrigLon"]
            dName  =  row["DestName"]
            dLat   =  row["DestLat"]
            dLon   =  row["DestLon"]
            floMag =  row["FlowMag"]

            thisRecordStrings = [oName, oLat, oLon, dName, dLat, dLon, floMag]
            thisOrigin = (float(thisRecordStrings[1]), float(thisRecordStrings[2]))
            if thisOrigin not in originGroups: # Make new dictionary entry if new.
                originGroups[thisOrigin] = []
                originKeys.append(thisOrigin)
            # Whether new or not, append this record to the values of its key.
            originGroups[thisOrigin].append(thisRecordStrings)

        for ok in originKeys:

            theseArcs = originGroups[ok]

            for a in theseArcs:

                if verbose:
                    print(str(a[0]) + " to " + str(a[3]) + "..." )

                originLatLon = ok # lat, lon
                destinLatLon = (float(a[4]), float(a[5])) # lat, lon

                # Convert these lat lon pairs to x,y in the outbound projected coordinate system, using pyproj.
                xOrigOut, yOrigOut = pOut(originLatLon[1], originLatLon[0])
                xDestOut, yDestOut = pOut(destinLatLon[1], destinLatLon[0])

                origMapVert = (xOrigOut, yOrigOut)
                destMapVert = (xDestOut, yDestOut)

                ## Find the "dev" point for defining an interpolator, using vector geometry.

                # Straight-line route as a vector starting at coord system origin is second vertex minus first.
                routeVector = np.array([destMapVert[0], destMapVert[1]]) - np.array([origMapVert[0], origMapVert[1]])

                # get along-track fraction of line as vector.
                alongTrackVector = routeVector * alongSegmentFraction

                # The user-set fraction of the arc distance for point dev.
                deviationVector = routeVector * devFraction

                # Get the left-handed orthogonal vector of this.
                orthogVector = calcOrthogonalVector(deviationVector, clockWise)

                # dev point is at the origin point + aMidpointVector + orthogVector
                devPointVector = np.array([origMapVert[0], origMapVert[1]]) + alongTrackVector + orthogVector
                devMapVert = (devPointVector[0], devPointVector[1])

                # Now determine the interpolator going through the origin, the dev point, and the destination.
                # NB: Usually, for the SciPy functions we use, the x values must be a strictly monotonic,
                # increasing series. To handle all cases, we will translate all three points equally so that the
                # origin point lies on the coordinate system origin, and rotate all points counterclockwise so
                # that the origin and destination y values are both 0. This will ensure the three x values are
                # monotonic, increasing in sequence.

                # Translate all points by negative vector of origMapVert, so origMapVert lies on the origin.
                orgV = np.array([origMapVert[0], origMapVert[1]])
                devV = np.array([devMapVert[0], devMapVert[1]])
                desV = np.array([destMapVert[0], destMapVert[1]])
                orgV_shft = np.array([0.0, 0.0]) # orgV_shft minus itself.
                devV_shft = devV - orgV
                desV_shft = desV - orgV
                devPt = Point(devV_shft[0], devV_shft[1]) # Shapely Point object.
                desPt = Point(desV_shft[0], desV_shft[1]) # Shapely Point object.
                # Determine angle necessary to rotate desV_shft so it lies on the x axis.
                # The origin vertex obviously doesn't change, but the other two do.
                # Angle of rotation necessary is given in radians by math.atan2(y2-y1, x2-x1).
                # Thanks to Jim Lewis: http://stackoverflow.com/questions/2676719/calculating-the-angle-between-the-line-defined-by-two-points
                theta_desV_shift = math.atan2( desV_shft[1] , desV_shft[0] ) # Returned in radians.
                angleToRotateBy = -1.0 * theta_desV_shift
                # Rotate both the dev point and the destination point by this angle.
                orgV_shft_rot = orgV_shft # Origin unchanged.
                devV_shft_rot = aff.rotate(devPt, angleToRotateBy, origin = (0.0, 0.0), use_radians = True)
                desV_shft_rot = aff.rotate(desPt, angleToRotateBy, origin = (0.0, 0.0), use_radians = True)
                # Restate each point as a simple tuple.
                orgV_shft_rot_tuple = (0.0, 0.0)
                devV_shft_rot_tuple = (devV_shft_rot.x, devV_shft_rot.y)
                desV_shft_rot_tuple = (desV_shft_rot.x, desV_shft_rot.y)
                # We've got the three necessary vertices to construct an interpolator, now in strictly increasing x order.
                interpoVerts = [orgV_shft_rot_tuple, devV_shft_rot_tuple, desV_shft_rot_tuple]
                #
                # Just a sanity check...
                if not strictly_increasing([ orgV_shft_rot_tuple[0], devV_shft_rot_tuple[0], desV_shft_rot_tuple[0] ]):
                    print("X values for this interpolation are not strictly increasing!")
                # The interpolator:
                series_x = [i[0] for i in interpoVerts]
                series_y = [i[1] for i in interpoVerts]
                thisInterpolator = generateInterpolator(series_x, series_y, interpolator)

                # Determine how many vertices each arc should have, using user-specified vertsPerArc,
                # over the range defined by the destination x - the origin x.
                xRange = series_x[2] - series_x[0]
                anInterval = xRange / vertsPerArc
                # xValues = np.linspace(series_x[0], series_x[2], num=anInterval, endpoint=True) # works, but slower by far than np.append()
                xValues = np.append( np.arange(series_x[0], series_x[2], anInterval), series_x[2] )
                # NB: This leaves the dev point behind! We should have many others near it though,
                # or it could be inserted into the sequence here.
                #
                # Add final (rotated and translated) destination x value to xValues.
                np.append(xValues, desV_shft_rot_tuple[0])
                # Evaluate interpolants by thisInterpolator([xValues]), store vertices as tuples (x,y).
                yValues = thisInterpolator(xValues)
                # Build list of verts with origin at beginning, then interpolated ones, then destination.
                vertsInterpolated = [ (x,y) for x,y in zip(xValues, yValues) ]
                # Now rotate these points back...
                rerotatedPoints = []
                for vi in vertsInterpolated:
                    aVert = Point(vi[0], vi[1]) # Shapely Point object.
                    aRerotatedPoint = aff.rotate(aVert, theta_desV_shift, origin = (0.0, 0.0), use_radians = True)
                    rerotatedPoints.append(aRerotatedPoint)
                # ...and now translate the rerotated points back to projected map coordinates.
                rectifiedPoints = []
                for rrp in rerotatedPoints:
                    rrpV = np.array([rrp.x, rrp.y])
                    rectV = rrpV + orgV
                    aPoint = (rectV[0], rectV[1])
                    rectifiedPoints.append(aPoint)
                # Finally, build a line with this list of vertices, carrying over
                # the FlowMag attribute, and write to file.
                anArc = ogr.Feature(layer_defn)
                anArc.SetField( textFieldNames[0], a[0]) # origin
                anArc.SetField( textFieldNames[1], a[3]) # destination
                anArc.SetField(floatFieldNames[0], a[6]) # flow
                anArc.SetField(floatFieldNames[1], a[1]) # origin lat
                anArc.SetField(floatFieldNames[2], a[2]) # origin lon
                anArc.SetField(floatFieldNames[3], a[4]) # destination lat
                anArc.SetField(floatFieldNames[4], a[5]) # destination lon
                lineGeometry = createLineString(rectifiedPoints) # actually create the line
                anArc.SetGeometry(lineGeometry)
                dst_layer.CreateFeature(anArc)
                anArc = None # Free resources, finish this route.

    dst_ds = None # Destroy the data source to free resouces and finish writing.

    print("Finished, output written to: " + outFile)

# Main module check /////////////////////////////////////////////////////////////////////

if __name__ == '__main__':
    main()

# fin


from time import sleep
import os

class Player:
    # str, Room(class reference)
    #{name: 'Jackee', location: SOME OBJECT REFERENCE}
        def __init__(self, name, startRoom):
            self.room = startRoom
            self.name = name
            self.contents = []
            self.contentsHistory = []
            self.score = 0
        def move_to(self, direction):
            cardinal = direction + '_to'
            # if hasattr(self.location, 'cardinal'):
            #     #move towards it --->
            #     self.location = self.location.__dict__[cardinal]
            # else:
            #     os.system('cls' if os.name == 'nt' else 'clear')
            #     print('\nPick a different option, You cannot go that way...')
            #     sleep(5)
            try:
                if getattr(self.location, cardinal):
                    #move towards it --->
                    self.location = self.location.__dict__[cardinal]
            except AttributeError:
                if direction == 'q':
                    os.system('cls' if os.name == 'nt' else 'clear')
                    print('Thank you for playing..')
                    sleep(1)
                    os._exit(0)
                os.system('cls' if os.name == 'nt' else 'clear')
                print('\nPick a different option, You cannot go that way...')
                sleep(3)


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /srv/RunestoneComponents/runestone/common/runestonedirective.py
# Compiled at: 2020-04-16 12:40:55
# Size of source mod 2**32: 14669 bytes
__author__ = 'bmiller'
from collections import defaultdict
import binascii, os
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst import Directive
from docutils.utils import get_source_line
from docutils.statemachine import ViewList
from sphinx import application
from sphinx.errors import ExtensionError
UNNUMBERED_DIRECTIVES = [
 'activecode',
 'reveal',
 'video',
 'youtube',
 'vimeo',
 'codelens',
 'showeval',
 'poll',
 'tabbed',
 'tab',
 'timed',
 'disqus']

class RunestoneNode(nodes.Node):
    pass

class Struct:

    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __repr__(self):
        args = ['{}={}'.format(k, repr(v)) for k, v in vars(self).items()]
        return 'Struct({})'.format(', '.join(args))

def _get_runestone_data(env):
    if not hasattr(env, 'runestone_data'):
        env.runestone_data = Struct(id_to_page=(dict()),
          page_to_id=(defaultdict(set)))
    return env.runestone_data

def _purge_runestone_data(app, env, docname):
    runestone_data = _get_runestone_data(env)
    for id_ in runestone_data.page_to_id[docname]:
        runestone_data.id_to_page.pop(id_)

    runestone_data.page_to_id.pop(docname)
    env.assesscounter = 0

def _add_autoversion(self, filename):
    for path in self.config._raw_config['html_static_path']:
        full_path = path
        if not os.path.isabs(full_path):
            full_path = os.path.join(self.confdir, full_path)
        if os.path.isdir(path):
            full_path = os.path.join(path, filename)
        else:
            if os.path.normpath(path).endswith(os.path.normpath(filename)):
                full_path = path
            else:
                continue
            try:
                f = open(os.path.join(path, filename), 'rb')
            except IOError:
                continue
            else:
                with f:
                    crc_str = '{:02X}'.format(binascii.crc32(f.read()))
                return '{}?v={}'.format(filename, crc_str)

    raise ExtensionError('Unable to find {} in html_static_path.'.format(filename))

def _add_autoversioned_javascript(self, filename):
    return self.add_javascript(self.add_autoversion(filename))

def _add_autoversioned_stylesheet(self, filename, *args, **kwargs):
    return (self.add_stylesheet)(self.add_autoversion(filename), *args, **kwargs)

application.Sphinx.add_autoversion = _add_autoversion
application.Sphinx.add_autoversioned_javascript = _add_autoversioned_javascript
application.Sphinx.add_autoversioned_stylesheet = _add_autoversioned_stylesheet

def setup(app):
    app.connect('env-purge-doc', _purge_runestone_data)
    app.add_role('skipreading', SkipReading)
    app.add_config_value('runestone_server_side_grading', False, 'env')
    app.add_config_value('generate_component_labels', True, 'env')

class RunestoneDirective(Directive):
    option_spec = {'author':directives.unchanged,
     'tags':directives.unchanged,
     'difficulty':directives.positive_int,
     'autograde':directives.unchanged,
     'practice':directives.unchanged,
     'topics':directives.unchanged}

    def __init__(self, *args, **kwargs):
        (super(RunestoneDirective, self).__init__)(*args, **kwargs)
        env = self.state.document.settings.env
        self.srcpath = env.docname
        split_docname = self.srcpath.split('/')
        if len(split_docname) < 2:
            split_docname.append('')
        self.subchapter = split_docname[(-1)]
        self.chapter = split_docname[(-2)]
        self.basecourse = self.state.document.settings.env.config.html_context.get('basecourse', 'unknown')
        self.options['basecourse'] = self.basecourse
        self.options['chapter'] = self.chapter
        self.options['subchapter'] = self.subchapter

class RunestoneIdDirective(RunestoneDirective):

    def getNumber(self):
        env = self.state.document.settings.env
        if self.name in UNNUMBERED_DIRECTIVES or env.config.generate_component_labels is False:
            return ''
        env.assesscounter += 1
        res = 'Q-%d'
        if hasattr(env, 'assessprefix'):
            res = env.assessprefix + '%d'
        res = res % env.assesscounter
        if hasattr(env, 'assesssuffix'):
            res += env.assesssuffix
        return res

    def updateContent(self):
        if self.content:
            if self.content[0][:2] == '..':
                self.content = ViewList([
                 self.options['qnumber'] + ':', ''], self.content.source(0)) + self.content
            else:
                if self.options['qnumber']:
                    self.content[0] = self.options['qnumber'] + ': ' + self.content[0]

    def run(self):
        if not self.required_arguments >= 1:
            raise AssertionError
        else:
            if 'divid' not in self.options:
                id_ = self.options['divid'] = self.arguments[0]
            else:
                id_ = self.options['divid']
            self.options['qnumber'] = self.getNumber()
            env = self.state.document.settings.env
            runestone_data = _get_runestone_data(env)
            id_to_page = runestone_data.id_to_page
            page_to_id = runestone_data.page_to_id
            if id_ in id_to_page:
                page = id_to_page[id_]
                if page.docname != env.docname or page.lineno != self.lineno:
                    raise self.error('Duplicate ID -- see {}, line {}'.format(page.docname, page.lineno))
                assert id_ in page_to_id[page.docname]
            else:
                id_to_page[id_] = Struct(docname=(env.docname), lineno=(self.lineno))
                page_to_id[env.docname].add(id_)

def first_time(app, *keys):
    key = '$'.join(keys)
    if not hasattr(app, 'runestone_flags'):
        app.runestone_flags = set()
    if key not in app.runestone_flags:
        app.runestone_flags.add(key)
        return True
    return False

def add_i18n_js(app, supported_langs, *i18n_resources):
    if first_time(app, 'add_i18n_js'):
        app.add_autoversioned_javascript('jquery_i18n/CLDRPluralRuleParser.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.messagestore.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.fallbacks.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.language.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.parser.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.emitter.js')
        app.add_autoversioned_javascript('jquery_i18n/jquery.i18n.emitter.bidi.js')
    for res in i18n_resources:
        if first_time(app, 'add_i18n_js', res):
            app.add_autoversioned_javascript(res + '.en.js')
            if app.config.language and app.config.language != 'en' and app.config.language in supported_langs:
                app.add_autoversioned_javascript(res + '.' + app.config.language + '.js')

def add_skulpt_js(app):
    if first_time(app, 'add_skulpt_js'):
        app.add_autoversioned_javascript('skulpt.min.js')
        app.add_autoversioned_javascript('skulpt-stdlib.js')
        app.add_javascript('https://cdn.jsdelivr.net/npm/vega@4.0.0-rc.2/build/vega.js')
        app.add_javascript('https://cdn.jsdelivr.net/npm/vega-lite@2.5.0/build/vega-lite.js')
        app.add_javascript('https://cdn.jsdelivr.net/npm/vega-embed@3.14.0/build/vega-embed.js')

def get_node_line(node):
    return get_source_line(node)[1]

def SkipReading(roleName, rawtext, text, lineno, inliner, options={}, content=[]):
    docname = inliner.document.settings.env.docname
    if not hasattr(inliner.document.settings.env, 'skipreading'):
        inliner.document.settings.env.skipreading = set()
    print('ADDING {} to skipreading'.format(docname))
    inliner.document.settings.env.skipreading.add(docname)
    return ([], [])


import csv

from dashboard.models import Play, Penalty

from common.views import get_team_by_name, EXIT_CONDITIONS

__author__ = 'dgosting'

def parse_play_by_play_data(absolute_path):

    with open(absolute_path, 'rtU') as csvfile:
        reader = csv.reader(csvfile)
        next(reader)
        plays = []
        counter = 0
        for row in reader:
            description = row[10]

            if any(exit_condition in description for exit_condition in EXIT_CONDITIONS):
                continue
            new_play = Play(game_id=int(row[0]),
                            date=format_date(row[0]),
                            quarter=int(row[2]),
                            minute=int(row[3]),
                            second=(row[4]),
                            offensive_team=get_team_by_name(row[5]),
                            defensive_team=get_team_by_name(row[6]),
                            down=int(row[7]),
                            yards_to_go=int(row[8]),
                            yard_line=int(row[9]),
                            description=description,
                            yards_gained=int(row[11]),
                            formation=row[12],
                            play_type=row[13],
                            is_incomplete=True if int(row[14]) == 1 else False,
                            is_touchdown=True if int(row[15]) == 1 else False,
                            direction=_get_play_direction(row),
                            is_sack=True if int(row[17]) == 1 else False,
                            is_interception=True if int(row[18]) == 1 else False,
                            is_fumble=True if int(row[19]) == 1 else False,
                            penalty=_get_penalty(row),
                            is_two_point_conversion=True if int(row[21]) == 1 else False,
                            is_two_point_conversion_successful=True if int(row[22]) == 1 else False,
                            is_no_play=True if int(row[26]) == 1 else False)

            plays.append(new_play)
            counter += 1
            if counter >= 1000:
                Play.objects.bulk_create(plays)
                plays = []
                counter = 0
        else:
            Play.objects.bulk_create(plays)

def delete_all_plays():

    Play.objects.all().delete()

def _get_penalty(row):

    if int(row[20]) == 1:
        return Penalty(is_accepted=int(row[24]),
                       penalized_team=get_team_by_name(row[25]),
                       description=row[27],
                       yards=int(row[28]))
    else:
        return None

def _get_play_direction(row):

    if row[13] == 'PASS':

        return row[16]

    elif row[13] == 'RUSH':

        return row[23]

    return 'None'

def format_date(date):

    formatted_date = date[:4] + "-" + date[4:6] + "-" + date[6:8]

    return formatted_date

def count_plays():

    print(Play.objects.all().count())


# -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function

import numpy as np

import torch
import torch.nn.functional as F

'''
Average:
An utterance representation can be obtained by averaging the embeddings of all the words in that utterance, of which the cosine similarity gives the Average metric
'''

def get_top_k_fact_average_batch(encoder_embedding, fact_embedding, encoder_embedding_size,
                                fact_embedding_size, encoder_inputs, fact_inputs,
                                batch_size=1, top_k=20, device=None):

    encoder_inputs_embedded = encoder_embedding(encoder_inputs)
    encoder_inputs_embedded_mean = encoder_inputs_embedded.transpose(0, 1).mean(dim=1)  # [batch_size, embedding_size]

    new_fact_inputs = torch.zeros((batch_size, top_k, fact_embedding_size), device=device)
    for bi in range(batch_size):
        cur_fact_input = fact_inputs[bi]
        # cur_fact_input: [total_n, len()]
        cur_fact_input_embedded_mean = torch.zeros(
            (len(cur_fact_input), fact_embedding_size), device=device)
        for fi, cur_part_fact in enumerate(cur_fact_input):
            cur_part_fact = torch.LongTensor(cur_part_fact, device=device)
            cur_part_fact_embedded = fact_embedding(cur_part_fact.view(1, -1))
            cur_part_fact_embedded_mean = cur_part_fact_embedded.mean(dim=1)
            cur_fact_input_embedded_mean[fi] = cur_part_fact_embedded_mean.view(-1)

        # get top_k
        cur_encoder_input_embedded_mean = encoder_inputs_embedded_mean[bi]
        cosine_scores = F.cosine_similarity(
            cur_encoder_input_embedded_mean, cur_fact_input_embedded_mean)

        # sort
        sorted_scores, sorted_indices = cosine_scores.sort(
            dim=0, descending=True)
        top_k_indices = sorted_indices[:top_k]
        # [top_k, embedding_size]
        top_k_fact_embedded = cur_fact_input_embedded_mean[top_k_indices]
        new_fact_inputs[bi] = top_k_fact_embedded

    return new_fact_inputs

'''without batch'''

def get_top_k_fact_average(encoder_embedding, fact_embedding, encoder_embedding_size,
                           fact_embedding_size, conversation_ids, facts_ids, facts_weight,
                           top_k=20, device=None):
    """
    Args:
        - conversation_ids: [len]
        - facts_ids: total_num, len
    """
    with torch.no_grad():
        encoder_input = torch.tensor(conversation_ids, dtype=torch.long, device=device)
        encoder_input_embedded = encoder_embedding(encoder_input, is_dropout=False)
        encoder_input_embedded_mean = encoder_input_embedded.mean(dim=0).unsqueeze(0)  # [1, embedding_size]

        facts_embedded_mean = torch.zeros((len(facts_ids), fact_embedding_size), device=device)
        for fi, fact_ids in enumerate(facts_ids):
            fact_input = torch.tensor(fact_ids, dtype=torch.long, device=device)
            fact_input_embedded = fact_embedding(fact_input, is_dropout=False)
            fact_input_embedded_mean = fact_input_embedded.mean(dim=0)  # [embedding_size]
            facts_embedded_mean[fi] = fact_input_embedded_mean

        # get top_k
        cosine_scores = F.cosine_similarity(facts_embedded_mean, encoder_input_embedded_mean)  # [len(facts_ids)]

        cosine_scores = cosine_scores * facts_weight

        # * tag_weight

        # sort
        _, sorted_indices = cosine_scores.sort(dim=0, descending=True)

        top_k_indices = sorted_indices[:top_k]
        # [top_k, embedding_size]
        #  top_k_facts_embedded_mean = facts_embedded_mean[top_k_indices]

        top_k_facts_embedded_mean = torch.zeros((top_k, fact_embedding_size), device=device)
        for i in top_k_indices:
            top_k_facts_embedded_mean[i] = facts_embedded_mean[i]

        del facts_embedded_mean
        del encoder_input_embedded_mean
        del encoder_input_embedded
        del encoder_input
        del cosine_scores

    return top_k_facts_embedded_mean.detach().cpu(), top_k_indices.detach().cpu()

'''
Extreme:
Achieve an utterance representation by taking the
largest extreme values among the embedding vectors of all the words it contains
'''

def get_extreme_embedding_score(vocab, gensim_model, input_str, candidate_replies,
                                stop_word_obj, lower=None, normal=False):
    query_words = stop_word_obj.remove_words(input_str.strip().replace('\t', ' '))  # to lower if lower:
    query_words = [word.lower() for word in query_words]

    query_words_embedding = []
    for word in query_words:
        try:
            word_embedding = gensim_model.wv[word]
        except KeyError:
            word_embedding = gensim_model.wv[vocab.unk]
        query_words_embedding.append(word_embedding)

    extreme_vector_query = np.array(query_words_embedding).max(axis=0)

    extreme_matrix_candidate = []
    for candidate_reply in candidate_replies:
        candidate_words = stop_word_obj.remove_words(
            candidate_reply.strip().replace('\t', ' '))

        # to lower
        if lower:
            candidate_words = [word.lower() for word in candidate_words]

        candidate_words_embedding = []
        for word in candidate_words:
            try:
                word_embedding = gensim_model.wv[word]
            except KeyError:
                word_embedding = gensim_model.wv[vocab.unk]

            candidate_words_embedding.append(word_embedding)

        extreme_vector_candidate = np.array(
            candidate_words_embedding).max(axis=0)
        extreme_matrix_candidate.append(extreme_vector_candidate)

    extreme_matrix_candidate = np.array(extreme_matrix_candidate)

    score_vector = gensim_model.cosine_similarities(
        extreme_vector_query, extreme_matrix_candidate)

    # normalization
    if normal:
        score_vector = score_normalization(score_vector)

    return score_vector

'''
Greedy:
Greedily match words in two given utterances based on the cosine similarities of their embeddings,
and to average the obtained scores
'''

def get_greedy_embedding_score(vocab, gensim_model, input_str, candidate_replies, stop_word_obj, lower=None,
                               normal=False):
    query_words = stop_word_obj.remove_words(
        input_str.strip().replace('\t', ' '))

    # to lower
    if lower:
        query_words = [word.lower() for word in query_words]

    score_vector = []
    for candidate_reply in candidate_replies:
        candidate_words = stop_word_obj.remove_words(
            candidate_reply.strip().replace('\t', ' '))

        # to lower
        if lower:
            candidate_words = [word.lower() for word in candidate_words]

        max_scores = []
        for query_word in query_words:
            max_score = 0.0
            for candidate_word in candidate_words:
                try:
                    score = gensim_model.wv.similarity(
                        query_word, candidate_word)
                except KeyError:
                    score = -1.0

                max_score = max(score, max_score)

            max_scores.append(max_score)

        score_vector.append(np.mean(max_scores))

    # normalization
    if normal:
        score_vector = score_normalization(score_vector)

    return score_vector

'''
Optimal Matching
TODO
'''

'''
Word Mover's Distance
'''

def get_wmd_embedding_score(gensim_model, input_str, candidate_replies, stop_word_obj, lower=None, normal=False):
    distance_vector = np.random.rand(len(candidate_replies))

    query_words = stop_word_obj.remove_words(
        input_str.strip().replace('\t', ' '))
    # to lower
    if lower:
        query_words = [word.lower() for word in query_words]

    for idx, candidate_reply in enumerate(candidate_replies):
        candidate_words = stop_word_obj.remove_words(
            candidate_reply.strip().replace('\t', ' '))
        # to lower
        if lower:
            candidate_words = [word.lower() for word in candidate_words]

        distance_vector[idx] = gensim_model.wmdistance(
            query_words, candidate_words)

    # distance to score
    score_vector = np.max(distance_vector) - distance_vector

    # normalization
    if normal:
        score_vector = score_normalization(score_vector)

    return score_vector

'''
Max-Min normalization.
'''

def score_normalization(score_vector):
    max_score = np.max(score_vector)
    min_score = np.min(score_vector)
    score_vector = np.divide((score_vector - min_score),
                             (max_score - min_score) * 1.0)
    return score_vector


from xml.etree import ElementTree

class XMLReader(object):

    @staticmethod
    def getRootNodeFromFile(filePath):

        tree = ElementTree.parse(filePath)
        rootNode = tree.getroot()
        return rootNode

    @staticmethod
    def getChildElementsByTagName(rootNode, tagName):

        elementsToReturn = []

        for element in rootNode.iter(tagName):

            elementsToReturn.append(element)

        return elementsToReturn

    @staticmethod
    def getChildElementTextValuesByTagName(rootNode, tagName):

        elementsToReturn = []

        for element in rootNode.iter(tagName):
            elementsToReturn.append(element.text)

        return elementsToReturn

    @staticmethod
    def getSingleChildElementByTagName(rootNode, tagName):

        for element in rootNode.iter(tagName):

            return element

        return None


import logging
try:
        import cPickle as pickle
except:
        import pickle
import numpy as np
import zlib

logger  = logging.getLogger('FTStore')

class FTStore(object):

    def __init__(self, mode='MEM', ind='test', load=True):
        self.dirs  = '0_DB'
        self.store = None
        self.mode  = mode

        if mode == 'MEMNOZIP':
            self.store = dict()
        if mode == 'MEM':
            self.store = dict()
        if mode == 'ROCKS':
            import rocksdb
            if load is True:
                self.store = self.initRocks(ind)
            else:
                self.store = self.initRocks(ind)
        if mode == 'REDIS':
            import redis
            self.store = self.initRedis(ind)

    def addBatch(self, kvs):
        batch = rocksdb.WriteBatch()
        for kv in kvs:
            batch.put(self.wrap(kv[0]), self.wrap(kv[1]))
        self.store.write(batch, disable_wal=True)

    def wrap(self, string):
        return string

    def delete(self):
        del self.store
        self.store = None
        logger.info('DEL_STORE_SUCCESS')

    def update(self, store):
        if self.mode == 'MEM':
            self.store.update(store)
        if self.mode == 'ROCKS':
            self.store = store

    def add(self, key, val):
        if self.mode == 'MEM':
            val0 = np.asarray(val[0] * 10000000, int)
            cz   = zlib.compress(val0)
            self.store[key] = (cz, val[1])
        if self.mode == 'ROCKS':
            self.store.put(self.wrap(key), self.wrap(val), disable_wal=True)
        if self.mode == 'REDIS':
            val0 = np.asarray(val[0] * 10000000, int)
            cz   = zlib.compress(val0)
            self.store.set(key, (cz, val[1]))
        if self.mode == 'MEMNOZIP':
            self.store[key] = val

    def get(self, key):

        if self.mode == 'MEMNOZIP':
            return self.store[key]

        if self.mode == 'MEM':
            val  = self.store[key]
            dstr = zlib.decompress(val[0])
            dval = np.fromstring(dstr, dtype=int)
            dval = dval / 10000000.0
            del dstr
            dstr = None
            return (dval, val[1])

        if self.mode == 'ROCKS':
            logger.debug('Request key:{}'.format(key))
            vals = self.store.get(self.wrap(key))
            if vals is None:
                return None
            return pickle.loads(vals)

        if self.mode == 'REDIS':
            logger.debug('Redis_Get_{}'.format(key))
            val  = eval(self.store.get(key))
            dstr = zlib.decompress(val[0])
            dval = np.fromstring(dstr, dtype=int)
            dval = dval / 10000000.0
            del dstr
            dstr = None
            return (dval, val[1])

    def contains(self, key):
        ret = True
        if self.mode == 'MEM':
            if key not in self.store:
                ret = False
        if self.mode == 'MEMNOZIP':
            if key not in self.store:
                ret = False
        if self.mode == 'ROCKS':
            ret = self.store.key_may_exist(key)[0]
        if self.mode == 'REDIS':
            ret = r.exists(key)
        return ret

    def readRocks(self, ind):
        opts = rocksdb.Options()
        opts.create_if_missing = True
        opts.max_open_files = 300000
        opts.write_buffer_size = 1073741824
        opts.max_write_buffer_number = 20
        opts.target_file_size_base = 67108864
        opts.max_background_compactions = 8
        opts.max_background_flushes = 4

        opts.table_factory = rocksdb.BlockBasedTableFactory(
            filter_policy=rocksdb.BloomFilterPolicy(10),
            block_cache=rocksdb.LRUCache(2 * (1024 ** 3)),
            block_cache_compressed=rocksdb.LRUCache(500 * (1024 ** 2)))

        db = rocksdb.DB(self.dirs + '/' + ind + '.db', opts, read_only=True)
        return db

    def initRedis(self, ind):
        port = 0
        if ind == 'k3':
            port = 28898
        r = redis.StrictRedis(host='localhost', port=port, db=0)
        return r

    def initRocks(self, ind):
        opts = rocksdb.Options()
        opts.create_if_missing = True
        opts.max_open_files = 300000
        opts.write_buffer_size = 1073741824
        opts.max_write_buffer_number = 20
        opts.target_file_size_base = 67108864
        opts.max_background_compactions = 8
        opts.max_background_flushes = 4

        opts.table_factory = rocksdb.BlockBasedTableFactory(
            filter_policy=rocksdb.BloomFilterPolicy(10),
            block_cache=rocksdb.LRUCache(2 * (1024 ** 3)),
            block_cache_compressed=rocksdb.LRUCache(500 * (1024 ** 2)))

        db = rocksdb.DB(self.dirs + '/' + ind + '.db', opts)
        return db
#


from pygame import *

grid_pos = (0, 0)

def update(info):
    update_pos(info)

def update_pos(info):

    offset = info['window']['pos']

    pos = list(mouse.get_pos())
    offset = list(offset)

    pos[0] -= offset[0]
    pos[1] -= offset[1]

    pos[0] /= info['window']['game_scale']
    pos[1] /= info['window']['game_scale']

    pos[0] = int(pos[0])
    pos[1] = int(pos[1])

    global grid_pos
    grid_pos = list(pos)

def get_pos():
    global grid_pos
    return list(grid_pos)


#-*- coding: utf-8 -*-
import sys
import os
import time
import requests

#reload(sys)
#sys.setdefaultencoding('utf-8')

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.alert import Alert
from bs4 import BeautifulSoup

def bus_chk( start_id, dest_id, date) :
	print(start_id)
	print(dest_id)
	print(date)

	options = webdriver.ChromeOptions()
	options.add_argument('headless')
	options.add_argument('window-size=1920x1080')
	options.add_argument("disable-gpu")
	# 혹은 options.add_argument("--disable-gpu")

	ff_driver = webdriver.Chrome('/Users/peten/lib/chromedriver', chrome_options=options)

	#ff_driver = webdriver.Chrome('/Users/peten/lib/chromedriver')
	ff_driver.implicitly_wait(1)

	ff_driver.get("https://www.kobus.co.kr/main.do")

	#팝업메뉴제거
	elements = ff_driver.find_elements_by_css_selector("a[class='pop_close']")
	for i in elements:
		i.click()

	#버스 조회

	'''

	<input type="hidden" name="deprDtm" id="deprDtm" value="20190415">
	<input type="hidden" name="arvlDtm" id="arvlDtm" value="20190415">

	<input type="hidden" name="deprDtmAll" id="deprDtmAll" value="2019. 4. 15. 목">
	<input type="hidden" name="arvlDtmAll" id="arvlDtmAll" value="2019. 4. 15. 목">

	'''
	ff_driver.execute_script("document.getElementById('deprDtm').value = '20190517';")
	#ff_driver.execute_script("document.getElementById('arvlDtm').value = '20190228';")
	ff_driver.execute_script("document.getElementById('deprDtmAll').value='2019. 5. 17. 금';")
	#ff_driver.execute_script("document.getElementById('arvlDtmAll').value='2019. 3. 1. 금';")

	ff_driver.execute_script("fnReadDeprInfoList(event);")
	ff_driver.execute_script("fnDeprChc('010','서울경부');")
	ff_driver.execute_script("fnArvlChc('810','구미','N','','','180');")

	ff_driver.execute_script("fnDeprChc('810','구미');") #출발지
	ff_driver.execute_script("fnArvlChc('010','서울경부','N','','','240');") #도착지

	#ff_driver.execute_script("fnYyDtmStup(1,'text_date1','0');")
	ff_driver.execute_script("fnRotVldChc();")
	time.sleep(2)

	chk = 0
	try :
		alert = ff_driver.switch_to.alert
		alert.accept()
		ff_driver.quit()
		print('program end')
		chk = 1
	except :
		print('next')

	if chk == 1:
		exit()

	time.sleep(3)

	'''
	이제 각 터미널 번호와 터미널 이름을 매칭시켜 DB에 집어넣고
	DB를 구축해서 DB에 결과값을 집어넣는거 까지 하면 끝;

	몇자리 남았는지는 어떻게 확인하지??? (이건 사용자가 요청하면 확인하는 걸로;;)

	'''

	'''

	fnAlcnSrchBef();
	fnPopFeeAgrm();

	print(ff_driver.page_source)

	'''

	p_page = ff_driver.page_source
	ff_driver.quit()

	soup = BeautifulSoup(p_page, 'html.parser')
	notices = soup.select('div.bustime_wrap > div.bus_time > p > a > span.start_time')
	len_soup = len(notices)

	if len_soup == 0:
		print("조회 결과 :: 없음")
	else :
		print("select number : ", len_soup)

	for n in notices:
		print(n.text.strip())

x = input('Input x : ')
y = input('Input y : ')
z = input('Input date : ')
bus_chk(x,y,z)


from .UQAnalysis import UQAnalysis
from .RawDataAnalyzer import RawDataAnalyzer
from .Common import Common

class CorrelationAnalysis(UQAnalysis):

    def __init__(self, ensemble, output):
        self.moments = None
        super(CorrelationAnalysis, self).__init__(ensemble, output, UQAnalysis.CORRELATION)

    def analyze(self):
        data = self.ensemble.getValidSamples()
        Common.initFolder(RawDataAnalyzer.dname)
        fname = Common.getLocalFileName(RawDataAnalyzer.dname, data.getModelName().split()[0], '.dat')
        data.writeToPsuade(fname)

        #perform UA
        mfile = RawDataAnalyzer.performCA(fname, self.outputs[0])

        #archive file
        if mfile is not None:
            self.archiveFile(mfile)
        return mfile

    def showResults(self):
        #restore .m file from archive
        fileName = 'matlabca.m'
        self.restoreFromArchive(fileName)

        RawDataAnalyzer.plotCA(self.ensemble, self.outputs[0], fileName)


# Create a robot controller class. This is the mind of the robot.
# It should take an user input by listening to user input:
# Default functionaly when the robot is switched on:
#  - Automatically names the robot
#  - Sets it's position
#
# List of commands:
#  1) memorize: add a new memory entry to the memory
#  2) recall: displays a list memories
#  3) move: increments the robot's position by one coordinate the N-S-E-W directions
#     - it also displays the new position
#  4) speak: it can introduce itself by saying its name and mood and last memory
#
# Create the class with MVC pattern in mind. It should get and store data in the model object
# and it should pass the data to the view objects

import robot_model
import robot_view
import sys

class Control():

    def __init__(self):
        self.model = robot_model.Model()
        self.view = robot_view.View()

    def flow(self):
        if sys.argv[0] == "robot_controller.py" and len(sys.argv) < 2:
            self.printUsage()
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "n":
            self.move_N()
            self.view.draw_position(self.model.position_x, self.model.position_y, self.model.name, self.model.mood)
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "s":
            self.move_S()
            self.view.draw_position(self.model.position_x, self.model.position_y, self.model.name, self.model.mood)
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "e":
            self.move_E()
            self.view.draw_position(self.model.position_x, self.model.position_y, self.model.name, self.model.mood)
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "w":
            self.move_W()
            self.view.draw_position(self.model.position_x, self.model.position_y, self.model.name, self.model.mood)
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "m":
            self.memorize()
            #self.view.draw_position(self.model.position_x, self.model.position_y, self.model.name, self.model.mood)
        elif sys.argv[0] == "robot_controller.py" and sys.argv[1] == "l":
            self.list()

    def printUsage(self):
        start_screen = open('usage.txt', "r")
        for line in start_screen:
            print(line.rstrip())
        start_screen.close()

    def move_N(self):
        self.N = int(input("How far should I run in North direction? "))
        return self.model.m_N(self.N)

    def move_S(self):
        self.S = int(input("How far should I run in South direction? "))
        return self.model.m_S(self.S)

    def move_E(self):
        self.E = int(input("How far should I run in East direction? "))
        return self.model.m_E(self.E)

    def move_W(self):
        self.W = int(input("How far should I run in West direction? "))
        return self.model.m_W(self.W)

    def memorize(self):
        self.mem = str(input("What do you want me to remember? "))
        return self.model.remember(self.mem)

    def list(self):
        return self.model.memory_list()

robot = Control()
robot.flow()


import os,glob,re
from pymol import cmd
import simplevina_dev as SV
import multiprocessing
#====== input parameters here ======
#default_batch = 15
#default_extending = 5.0
#default_exhaustiveness = 8
#default_ncpu = 2
#vina_parameters_file = 'vina_parameters.txt'

extending, ncpu, exhaustiveness = 6.0, 4, 24
## ====== end input parameters =====

print("usage: run dock_one_sele_in_pymol.py  (in pymol cmd)\nnote:first need select organic(ligand) manually, cmd: 'alignto xx_first ,align' ;  and also need to cp simplevina_dev.py /home/phzd/2_software/pymol2/lib/python2.7/site-packages")

base = os.getcwd()

"""
def read_and_set_extending_ncpu_exhaustiveness(xx_file):
	os.chdir(base)
	if not os.path.exists(vina_parameters_file):
		print('warning !!, no vina_parameters_file was found, will use default setting for running vina')
		return '','',''
	f = open(xx_file,'r')
	txt = f.read()
	f.close()
	pattern1 = re.compile('extending: *(\d+\.*\d*)');	pattern2 = re.compile('--cpu *(\d+)');	pattern3 = re.compile('--exhaustiveness *(\d+)')
	match1 = re.findall(pattern1,txt);	match2 = re.findall(pattern2,txt);	match3 = re.findall(pattern3,txt)
	read_extending,read_ncpu,read_exhaustiveness = match1[0], match2[0], match3[0]
	extending = read_extending if read_extending != '' else default_extending
	ncpu = read_ncpu if read_ncpu != '' else default_ncpu
	exhaustiveness = read_exhaustiveness if read_exhaustiveness != '' else default_exhaustiveness
	print('inside os.getcwd(), funcion extending, ncpu, exhaustiveness =', os.getcwd(),extending, ncpu, exhaustiveness)
	return extending, ncpu, exhaustiveness
"""
base = os.getcwd()
print('base =',base)
os.chdir(base)

#extending, ncpu, exhaustiveness = read_and_set_extending_ncpu_exhaustiveness(vina_parameters_file)
print('extending, ncpu, exhaustiveness = ', extending, ncpu, exhaustiveness)

cmd.delete('box*')
cmd.delete('axes')
SV.docking_with_ligand_exp(selection = 'sele',extending = extending,exhaustiveness = exhaustiveness, ncpu = ncpu)
os.chdir(base)


# Copyright Contributors to the Pyro project.
# SPDX-License-Identifier: Apache-2.0

import pickle

import numpy as np
from numpy.testing import assert_allclose
import pytest

from jax import random
import jax.numpy as jnp
from jax.tree_util import tree_all, tree_map

import numpyro
import numpyro.distributions as dist
from numpyro.infer import (
    HMC,
    HMCECS,
    MCMC,
    NUTS,
    SA,
    SVI,
    BarkerMH,
    DiscreteHMCGibbs,
    MixedHMC,
    Predictive,
)
from numpyro.infer.autoguide import AutoDelta, AutoDiagonalNormal, AutoNormal

def normal_model():
    numpyro.sample("x", dist.Normal(0, 1))

def bernoulli_model():
    numpyro.sample("x", dist.Bernoulli(0.5))

def logistic_regression():
    data = jnp.arange(10)
    x = numpyro.sample("x", dist.Normal(0, 1))
    with numpyro.plate("N", 10, subsample_size=2):
        batch = numpyro.subsample(data, 0)
        numpyro.sample("obs", dist.Bernoulli(logits=x), obs=batch)

@pytest.mark.parametrize("kernel", [BarkerMH, HMC, NUTS, SA])
def test_pickle_hmc(kernel):
    mcmc = MCMC(kernel(normal_model), num_warmup=10, num_samples=10)
    mcmc.run(random.PRNGKey(0))
    pickled_mcmc = pickle.loads(pickle.dumps(mcmc))
    tree_all(tree_map(assert_allclose, mcmc.get_samples(), pickled_mcmc.get_samples()))

@pytest.mark.parametrize("kernel", [DiscreteHMCGibbs, MixedHMC])
def test_pickle_discrete_hmc(kernel):
    mcmc = MCMC(kernel(HMC(bernoulli_model)), num_warmup=10, num_samples=10)
    mcmc.run(random.PRNGKey(0))
    pickled_mcmc = pickle.loads(pickle.dumps(mcmc))
    tree_all(tree_map(assert_allclose, mcmc.get_samples(), pickled_mcmc.get_samples()))

def test_pickle_hmcecs():
    mcmc = MCMC(HMCECS(NUTS(logistic_regression)), num_warmup=10, num_samples=10)
    mcmc.run(random.PRNGKey(0))
    pickled_mcmc = pickle.loads(pickle.dumps(mcmc))
    tree_all(tree_map(assert_allclose, mcmc.get_samples(), pickled_mcmc.get_samples()))

def poisson_regression(x, N):
    rate = numpyro.sample("param", dist.Gamma(1.0, 1.0))
    batch_size = len(x) if x is not None else None
    with numpyro.plate("batch", N, batch_size):
        numpyro.sample("x", dist.Poisson(rate), obs=x)

@pytest.mark.parametrize("guide_class", [AutoDelta, AutoDiagonalNormal, AutoNormal])
def test_pickle_autoguide(guide_class):
    x = np.random.poisson(1.0, size=(100,))

    guide = guide_class(poisson_regression)
    optim = numpyro.optim.Adam(1e-2)
    svi = SVI(poisson_regression, guide, optim, numpyro.infer.Trace_ELBO())
    svi_result = svi.run(random.PRNGKey(1), 3, x, len(x))
    pickled_guide = pickle.loads(pickle.dumps(guide))

    predictive = Predictive(
        poisson_regression,
        guide=pickled_guide,
        params=svi_result.params,
        num_samples=1,
        return_sites=["param", "x"],
    )
    samples = predictive(random.PRNGKey(1), None, 1)
    assert set(samples.keys()) == {"param", "x"}


LEFT ="LEFT"
RIGHT ="RIGHT"
ECNODER_LEFT_PIN = 53
ECNODER_RIGHT_PIN = 51

MAX_THROTTEL = 0
MIN_THROTTEL = 0.8
OFF_THROTTLE = 1
STEP_SIZE = 0.5
FORWARD_STEP_SIZE=0.33

LEFT_MOTOR_PMW = 26
LEFT_MOTOR_GPIO = 20

RIGHT_MOTOR_PMW = 28
RIGHT_MOTOR_GPIO = 27


from matplotlib.pyplot import figure, plot, savefig, title, xlabel, ylabel, yticks
from numpy import genfromtxt, linspace
from sys import argv

input_file = str(argv[1])
output_file = str(argv[2])

data = genfromtxt(input_file, delimiter=',')
t = data[0]
norm = data[1]

fig = figure()
title('Norm of the Approximate Solution')
plot(t, norm, '-')
xlabel('t')
ylabel(r"$\Vert u(t)\Vert_{L^2(\mathbb{R}^2;\mathbb{C})}$")
yticks(linspace(0.0, 2.0 * max(norm), 11, endpoint=True))
savefig(output_file)

print('Generated ' + output_file)


import threading
import time
class MVar :

  def __init__(self) :

    self.content = None
    self.empty = True
    self.r = threading.Condition()
    self.w = threading.Condition()

  def take(self) :

    with self.r :
      while self.empty :
        self.r.wait()

      with self.w :
        self.w.notify()
        help = self.content
        self.content = None
        self.empty = True
        return help

  def put(self,v) :

    with self.w :
      while not self.empty :
        self.w.wait()

      with self.r :
        self.content = v
        self.empty = False
        self.r.notify()

class Chan :

  def __init__(self) :

    hole = MVar()
    self.r = MVar()
    self.r.put(hole)
    self.w = MVar()
    self.w.put(hole)

  def read(self) :

    r_end = self.r.take()
    v,next = r_end.take()
    self.r.put(next)
    return v

  def write(self,v) :

    hole = self.w.take()
    new_hole = MVar()
    hole.put((v,new_hole))
    self.w.put(new_hole)

  def is_empty(self) :

    r_end = self.r.take()
    self.r.put(r_end)
    w_end = self.w.take()
    self.w.put(w_end)
    return r_end == w_end

def producer(chanObj,n) :

  for i in range(n):
    chanObj.write(1)

def consumer(chanObj) :

  sum = 0
  content = None

  while content != -1 :
    content = chanObj.read()
    sum += content

  print(sum+1)

ch = Chan()

t1 = threading.Thread(target=producer, args=(ch,1000))
t2 = threading.Thread(target=producer, args=(ch,1000))
t3 = threading.Thread(target=producer, args=(ch,1000))
t4 = threading.Thread(target=producer, args=(ch,1000))
c1 = threading.Thread(target=consumer, args=(ch,))
c2 = threading.Thread(target=consumer, args=(ch,))
c3 = threading.Thread(target=consumer, args=(ch,))
c4 = threading.Thread(target=consumer, args=(ch,))

t1.start()
t2.start()
t3.start()
#t4.start()
c1.start()
c2.start()
c3.start()
c4.start()

input()

ch.write(-1)
ch.write(-1)
ch.write(-1)
ch.write(-1)