#!/usr/bin/env python
# coding: utf-8

# ## Grant Cloud
# ## 903194929
# ## CS-4641 HW3

# ### Imports

# In[1]:

import numpy as np
import matplotlib.pyplot, os.path
from mpl_toolkits.mplot3d import Axes3D

#
Provided code, with my modified functions

# In[2]:

## Data loading utility functions
def get_test_train(fname,seed,datatype):
	'''
	Returns a test/train split of the data in fname shuffled with
	the given seed

	Args:
		fname: 		A str/file object that points to the CSV file to load, passed to
					np.genfromtxt()
		seed:		The seed passed to np.random.seed(). Typically an int or long
		datatype:	The datatype to pass to genfromtxt(), usually int, float, or str

	Returns:
		train_X:	A NxD np array of training data (row-vectors), 80% of all data
		train_Y:	A Nx1 np array of class labels for the training data
		test_X:		A MxD np array of testing data, same format as train_X, 20% of all data
		test_Y:		A Mx1 np array of class labels for the testing data
	'''
	data = np.genfromtxt(fname,delimiter=',',dtype=datatype)
	np.random.seed(seed)
	shuffled_idx = np.random.permutation(data.shape[0])
	cutoff = int(data.shape[0]*0.8)
	train_data = data[shuffled_idx[:cutoff]]
	test_data = data[shuffled_idx[cutoff:]]
	train_X = train_data[:,:-1].astype(float)
	train_Y = train_data[:,-1].reshape(-1,1)
	test_X = test_data[:,:-1].astype(float)
	test_Y = test_data[:,-1].reshape(-1,1)
	return train_X, train_Y, test_X, test_Y

def load_HTRU2(path='data'):
	return get_test_train(os.path.join(path,'HTRU_2.csv'),seed=1567708903,datatype=float)

def load_iris(path='data'):
	return get_test_train(os.path.join(path,'iris.data'),seed=1567708904,datatype=str)

## The "digits" dataset has a pre-split set of data, so we won't do our own test/train split
def load_digits(path='data'):
	train_data = np.genfromtxt(os.path.join(path,'optdigits.tra'),delimiter=',',dtype=float)
	test_data = np.genfromtxt(os.path.join(path,'optdigits.tes'),delimiter=',',dtype=float)
	return train_data[:,:-1], train_data[:,-1].reshape(-1,1), test_data[:,:-1], test_data[:,-1].reshape(-1,1)

## You can use this dataset to debug your implementation
def load_test2(path='data'):
	return get_test_train(os.path.join(path,'data2.dat'),seed=1568572211, datatype=float)

class PCA():
	'''
	A popular feature transformation/reduction/visualization method

	Uses the singular value decomposition to find the orthogonal directions of
	maximum variance.
	'''
	def __init__(self):
		'''
		Initializes some data members to hold the three components of the
		SVD.
		'''
		self.u = None
		self.s = None
		self.v = None
		self.shift = None
		self.data = None

	def find_components(self,data):
		'''
		Finds the SVD factorization and stores the result.

		Args:
			data: A NxD array of data points in row-vector format.
		'''
		self.data = data
		self.shift = self.data - np.mean(data, axis=0) # normalized data
		self.u, self.s, self.v = np.linalg.svd(self.shift, compute_uv=True, full_matrices=False)
		self.v = self.v.T

	def transform(self,n_components,data=None):
		'''
		Uses the values computed and stored after calling find_components()
		to transform the data into n_components dimensions.

		Args:
			n_components: The number of dimensions to transform the data into.
			data: 	the data to apply the transform to. Defaults to the data
					provided on the last call to find_components()

		Returns:
			transformed_data: 	a Nx(n_components) array of transformed points,
								in row-vector format.
		'''
		if data is None:
			data = self.data
		else:
			self.find_components(data)
		return np.dot(data, self.v[:,:n_components])

	def inv_transform(self,n_components,transformed_data):
		'''
		Inverts the results of transform() (if given the same arguments).

		Args:
			n_components:		Number of components to use. Should match
								the dimension of transformed_data.
			transformed_data:	The data to apply the inverse transform to,
								should be in row-vector format

		Returns:
			inv_tform_data:		a NxD array of points in row-vector format
		'''
		#NOTE: Don't forget to "un-center" the data
		tX = np.dot(transformed_data, self.v[:,:n_components].T)
		return tX + np.mean(self.data, axis=0)

	def reconstruct(self,n_components,data=None):
		'''
		Casts the data down to n_components dimensions, and then reverses the transform,
		returning the low-rank approximation of the given data. Defaults to the data
		provided on the last call to find_components().
		'''
		return self.inv_transform(n_components,self.transform(n_components,data))

	def reconstruction_error(self,n_components,data=None):
		'''
		Useful for determining how much information is preserved in n_components dimensions.
		'''
		if data is None:
			data = self.data
		return np.linalg.norm(data-self.reconstruct(n_components,data),ord='fro')

	def plot_2D_proj(self,data=None,labels=None):
		'''
		Creates a 2D visualization of the data, returning the created figure. See
		the main() function for example usage.
		'''
		fig = matplotlib.pyplot.figure()
		proj_2d_data = self.transform(2,data)
		fig.gca().scatter(proj_2d_data[:,0],proj_2d_data[:,1],c=labels)
		fig.gca().set_title('PCA 2D transformation')
		return fig
	def plot_3D_proj(self,data=None,labels=None):
		'''
		Creates a 3D visualization of the data, returning the created figure. See
		the main() function for example usage.
		'''
		fig = matplotlib.pyplot.figure()
		ax = fig.add_subplot(111,projection='3d')
		proj_3d_data = self.transform(3,data)
		ax.scatter(proj_3d_data[:,0],proj_3d_data[:,1],proj_3d_data[:,2],c=labels)
		fig.gca().set_title('PCA 3D transformation')
		return fig

# You may find this method useful for providing a canonical ordering of cluster centers in KMeans.cluster
def consistent_ordering(array):
	rowvec_dists = np.linalg.norm(array,axis=1)
	dist_order = np.argsort(rowvec_dists,kind='stable')
	return array[dist_order,:]
class KMeans():
	'''
	A simple iterative clustering method for real-valued feature spaces.

	Finds cluster centers by iteratively assigning points to clusters and re-computing
	cluster center locations. Provided code expects self.clusters to contain the
	cluster centers in row-vector format.
	'''
	def __init__(self):
		'''
		Initializes data members.
		'''
		self.clusters = None
		self.print_every = 1000

	def cluster_distances(self,data):
		'''
		Computes the distance from each row of data to the cluster centers. Must call KMeans.cluster(), or
		otherwise set self.clusters first.

		Args:
			data:	The data to compute distances for, in row-vector format. Each row should have the
					same number of columns as each cluster

		Returns:
			dists:	A Nx(len(clusters)) array, one row for each row in data, one column for each
					cluster center, containing the distance for each point to each cluster
		'''
		return np.hstack([np.linalg.norm(data-c,axis=1).reshape(-1,1) for c in self.clusters])

	def cluster_label(self,data):
		'''
		Returns the label of the closest cluster to each row in data. Note that these labels are
		arbitrary, and do *not* correspond directly with class labels.

		Args:
			data:	Data to compute cluster labels for, in row-vector format.

		Returns:
			c_labels:	A N-by-1 array, one row for each row in data, containing the integer
						corresponding to the cluster closest to each point.
		'''
		return np.argmin(self.cluster_distances(data),axis=1)

	def cluster(self,data,k):
		'''
		Implements the k-Means iterative algorithm. Cluster centers are initially chosen randomly,
		then on each iteration each data point is assigned to the closest cluster center, and then the
		cluster centers are re-computed by averaging the points assigned to them. After finishing,
		self.clusters should contain a k-by-D array of the cluster centers in row-vector format.

		Args:
			data:	Data to be clustered in row-vector format. A N-by-D array.
			k:		The number of clusters to find.
		'''
		#This line should pick the initial clusters at random from the provided
		#data.
		self.clusters = data[np.random.choice(data.shape[0],k,replace=False),:]
		#An example of how to use the consistent_ordering() function, which you
		#may want to use to help determine if cluster centers have changed from one
		#iteration to the next
		self.clusters = consistent_ordering(self.clusters)
		#We know that k-Means will always converge, but depending on the initial
		#conditions and dataset, it may take a long time. For debugging purposes
		#you might want to set a maximum number of iterations. When implemented
		#correctly, none of the provided datasets take many iterations to converge
		#for most initial configurations.
		not_done = True
		itr = 0
		while not_done:
			itr += 1
			new_clusters = np.zeros(self.clusters.shape)
			#assign points to nearest cluster
			labels = self.cluster_label(data)
			#re-compute cluster centers by averaging the points assigned to them
			for i in range(new_clusters.shape[0]):
				new_clusters[i] = data[np.argwhere(labels==i)].mean(axis=0)
			#determine if clusters have changed from the previous iteration
			new_clusters = consistent_ordering(new_clusters)
			if np.array_equal(self.clusters, new_clusters):
				not_done = False
			#For debugging, print out every so often.
			if itr == 0:
				print('')
			if itr % self.print_every == 0:
				print("Iteration {}, change {}".format(itr,np.linalg.norm(new_clusters-self.clusters,ord='fro')))
			self.clusters = new_clusters
		print("Converged after {} iterations".format(itr))

	def normalized_mutual_information(self, data, labels):
		'''
		Since cluster assignments are not the same as class labels, we can't always directly
		compare them to measure clustering performance. However, we can measure the Mutual Information
		between two labelings, to see if they contain the same statistical information. This method
		implements the "Normalized Mutual Information Score" as described here:

		https://scikit-learn.org/stable/modules/clustering.html#mutual-information-based-scores

		Note that this version uses arithmetic mean, when comparing output with
		sklearn.metrics.mutual_info_score()
		'''
		cluster_labels = self.cluster_label(data)
		P_cl = np.zeros(len(self.clusters))
		P_gt = np.zeros(len(np.unique(labels)))
		P_clgt = np.zeros((len(P_cl),len(P_gt)))
		cl_masks = dict()
		gt_masks = dict()
		MI = 0.0
		H_cl = 0.0
		H_gt = 0.0
		for c_cl in range(len(P_cl)):
			cl_masks[c_cl] = (cluster_labels==c_cl).reshape(-1,1)
			P_cl[c_cl] = (cl_masks[c_cl]).astype(int).sum()/len(data)
			H_cl -= P_cl[c_cl]*np.log(P_cl[c_cl])
		for c_gt in range(len(P_gt)):
			gt_masks[c_gt] = labels == c_gt
			P_gt[c_gt] = (gt_masks[c_gt]).astype(int).sum()/len(data)
			H_gt -= P_gt[c_gt]*np.log(P_gt[c_gt])
		for c_cl in range(len(P_cl)):
			for c_gt in range(len(P_gt)):
				P_clgt[c_cl,c_gt] = (np.logical_and(cl_masks[c_cl], gt_masks[c_gt])).astype(int).sum()/len(data)
				if P_clgt[c_cl,c_gt] == 0.0:
					MI += 0
				else:
					MI += P_clgt[c_cl,c_gt]*np.log(P_clgt[c_cl,c_gt]/(P_cl[c_cl]*P_gt[c_gt]))
		return MI/(np.mean([H_cl,H_gt]))

	def plot_2D_clusterd(self, data, labels=None):
		'''
		Creates a 2D visualization of the data, returning the created figure. See
		the main() function for example usage.
		'''
		if self.clusters is None or len(self.clusters)!=2:
			self.cluster(data,2)
		fig = matplotlib.pyplot.figure()
		clusterd_2d_data = self.cluster_distances(data)
		fig.gca().scatter(clusterd_2d_data[:,0],clusterd_2d_data[:,1],c=labels)
		fig.gca().set_title('k-Means 2D cluster distance')
		return fig
	def plot_3D_clusterd(self, data, labels=None):
		'''
		Creates a 3D visualization of the data, returning the created figure. See
		the main() function for example usage.
		'''
		if self.clusters is None or len(self.clusters)!=3:
			self.cluster(data,3)
		fig = matplotlib.pyplot.figure()
		ax = fig.add_subplot(111,projection='3d')
		clusterd_3d_data = self.cluster_distances(data)
		ax.scatter(clusterd_3d_data[:,0],clusterd_3d_data[:,1],clusterd_3d_data[:,2],c=labels)
		fig.gca().set_title('k-Means 3D cluster distance')
		return fig

# ### 1a) and 1b)

# In[3]:

pca = PCA()
data = load_iris()
pca.find_components(data[0])
_,labels = np.unique(data[1],return_inverse=True)
pca.plot_2D_proj(data[0],labels)
pca.plot_3D_proj(data[0],labels)
print('') # prevents Jupyter from repeat printing a figure

# In[4]:

reconError = pca.reconstruction_error(2,data[0])
print('2d reconstruction error: ',reconError)
reconError = pca.reconstruction_error(3,data[0])
print('3d reconstruction error: ',reconError)

#
With the Iris dataset, we expect classification to perform better on the 2D transformed features than on the original dataset. From the 2D transformation plot we can see the purple class is easily distinguished from the other 2 classes, and the remaining 2 classes have very little overlap. This is because PCA projects onto new axis' built from the eigenvalues of the highest variance features in the original dataset (this process is called Truncated SVD), this leads to reduced noise in the transformed data. In this case a good learner would definitely improve performance since we lost little (if any) signal but removed a lot of noise.

#
For the 3D case, we expect classification to perform about the same or maybe slightly better on the PCA transformed data as with the original features. This is because the Iris dataset only has 4 features, meaning the 3 transformed features contain almost all the same information as the original dataset. The amount of improvement depends on how much noise was contained in the 4th feature not chosen by PCA, as well as how much of that noise was removed when the transformation occurred. From the plot the purple points are still easily classified (same as with the original features) but there looks to be more overlap in the yellow and blue classes than in the case of the 2D transformation. Overall I would expect a classifier to perform better on the 2D transformed data than the 3D transformed data (since the 3D transformed data has additional noise).

#
For both cases we have about the same reconstruction error, showing the added feature may hold little to no information (and if any it is probably noise).

# In[5]:

data = load_digits()
pca.find_components(data[0])
_,labels = np.unique(data[1],return_inverse=True)
pca.plot_2D_proj(data[0],labels)
pca.plot_3D_proj(data[0],labels)
print('')

# In[6]:

reconError = pca.reconstruction_error(2,data[0])
print('2d reconstruction error: ',reconError)
reconError = pca.reconstruction_error(3,data[0])
print('3d reconstruction error: ',reconError)

#
The 2D PCA transformation on the digits dataset would likely decrease classification performance. This is because the digits dataset has so many different class labels. When PCA transforms the data into the lower dimension, we lose information on a lot of features. With so many different class labels, there is bound to be different classes that have similar values for the 2 features PCA chose and the distinguishing factor between these classes might be contained in a feature that was "squished" during the transformation. This loss of signal would decrease classification performance compared to the original feature set that still contains the important information to distinguish between these classes.

#
For the 3D case, we expect a classifier to perform better on the 3D transformed data than the original dataset. This is because the PCA transformation removes a lot of noise but maintains the information needed for a classifier to perform well (it looks like the additional feature selected in the 3D case contained a lot of signal). It may be the case that the best PCA transformation for the digits dataset would be one in 4D or even higher, since there are so many features and class labels, and there may be important signal in a feature that we lose when we scale down into 3D. We would also expect a classifier to perform better on the 3D PCA transformed data than the 2D PCA transormed data.
#
Performance on the digits dataset may also depend on what type of classifier is used. In the previous homework we saw how the high number of classes caused the logistic regression "one-vs-all" approach struggled with a high number of class labels. This would lead to lower performance on this dataset than kNN on the 3D PCA transformed data, which appears to be, in general, well clustered by class label in the plot.

# In[7]:

data = load_HTRU2()
pca.find_components(data[0])
_,labels = np.unique(data[1],return_inverse=True)
pca.plot_2D_proj(data[0],labels)
pca.plot_3D_proj(data[0],labels)
print('')

# In[8]:

reconError = pca.reconstruction_error(2,data[0])
print('2d reconstruction error: ',reconError)
reconError = pca.reconstruction_error(3,data[0])
print('3d reconstruction error: ',reconError)

#
The 2D PCA transformation on the HTRU2 dataset would decrease classifier performance. From the 2D plot we see a lot of overlap between the two classes. This is occurs because PCA chooses the two highest variance features (with the underlying assumption that they provide the most information about the data), but this can be flawed when there is a high variance, noisy feature. This feature might be highly variant but actually provides little to no information about the underlying classes of the data points. This is also the most likely reason that we see both classes occuring at both ends of the plot and a lot of overlap between classes. A classifier would struggle to classify the PCA transformed data using noisy features that contain little to no actual information about the underlying classes.

#
In the 3D PCA transformed data we see a much more distinguishable seperation of classes. The additional feature selected by PCA clearly contains important information about the underlying class of each data point. Furthermore the transformation squished other less variant features that may contain a lot of noise. It's interesting to note that we have a higher reconstruction error and higher classifier performance with the 3D transformation. This is shows that the additional "information" lost when we project onto the additional axis is mostly noise and isn't all that useful for a classifier in practice.

# ### 2a) and 2b)

# In[9]:

km = KMeans()
data = load_iris()
_,labels = np.unique(data[1],return_inverse=True)
km.plot_2D_clusterd(data[0],labels)
km.plot_3D_clusterd(data[0],labels)
print('')

#
kMeans would be a better transformation than PCA in the 2D case because the clusters look to be perpindicular, whereas in the PCA transform they have similar shape and size. It's easy to distinguish them when the class labels are present but when those labels are not present it would be much harder to distinguish the different classes in the 2D PCA transformation. In general I would expect a classifier to perform better when there are less points with different class labels next to each other in the 2D visualization and minimal overlap between classes.

#
In the 3D case I would expect a classifier to perform about equally on the PCA transformed data as the kMeans transformed data. Both transformation have a pretty clear seperation of class labels with some small overlap between two of the classes which would potentially lead to some loss in a classifier.

# In[11]:

data = load_digits()
_,labels = np.unique(data[1],return_inverse=True)
km.plot_2D_clusterd(data[0],labels)
km.plot_3D_clusterd(data[0],labels)
print('')

#
For both the 2D and 3D case in the digits dataset, the PCA transformation should be preferred to the kMeans transformation in terms of expected classifier performance. In both 2D and 3D kMeans transformations we see a large amount of overlap between different class labels and the clusters of classes are loose and wide. While the 2D PCA transformation probably wouldn't help classifier performance because of so much overlap between classes, the clustering of class labels was still tighter, and the 3D PCA transformation had clear clusters by different class labels with much less overlap. Overall a classifier would really struggle with classifying the kMeans transformed data.

# In[12]:

data = load_HTRU2()
_,labels = np.unique(data[1],return_inverse=True)
print('2D : ', end='')
km.plot_2D_clusterd(data[0],labels)
print('3D : ', end='')
km.plot_3D_clusterd(data[0],labels)
print('')

#
In both the 2D and 3D cases PCA is absolutely the preferred transformation in terms of best classifier performance. The kMeans transformed data makes it virtually impossible to distinguish between class labels. There is so much overlap between different classes, a classifier would struggle to distinguish different classes. The 2D PCA transformation, while still not super helpful for classifier performance, still has slightly easier to distinguish classes than the 2D kMeans transformed data. The 3D PCA transformation has a clear seperability between classes, that seperability is non-existant in the 3D kMeans transformed data plot.

# ### 2c)

# In[13]:

km = KMeans()
data = load_iris()
_,labels = np.unique(data[1],return_inverse=True)
numClasses = len(np.unique(labels))
print('# of Classes: ',numClasses)
km.cluster(data[0],numClasses)

# changing dtype to float for mutual information scoring
i = 0
for classLabel in list(np.unique(data[1])):
    data[1][data[1]==classLabel] = i
    i += 1

print('Mutual Information Score: ',km.normalized_mutual_information(data[0],data[1].astype('float64')))

#
kMeans did an ok job of clustering by class label on the iris dataset. This is because there is overlap between different class labels in the Iris dataset. When kMeans assigns points to clusters the assigned cluster is the one with the smallest distance from the point. This causes similar points with different class labels to be a part of the same cluster, decreasing the mutual information score. In the case of the Iris dataset, the 0.591 mutual information score means there is a statistical relationship between class label and cluster assignments (both labelings contain similar information), but it is not a perfect indicator.

# In[16]:

km = KMeans()
data = load_digits()
_,labels = np.unique(data[1],return_inverse=True)
numClasses = len(np.unique(labels))
print('# of Classes: ',numClasses)
km.cluster(data[0],numClasses)
print('Mutual Information Score: ',km.normalized_mutual_information(data[0],data[1]))

#
kMeans did the best job clustering according to class labels on the digits dataset, with the highest mutual information score of 0.745 . This tells us that the clusters and labelings both contain similar information (there is a strong statistical relationship). We also notice that it took the most iterations to converge of all the datasets, this is because kMeans converges when cluster assignments no longer change from one iteration to the next. In the case of digits, there are 10 class labels. The high number of clusters means that more points are likely to be reassigned from one iteration to the next (since there are more potential assignments). Digits has such a high mutual information score because data points that have the same class label have similar distances to the different cluster centers (mean of all the points assigned to a cluster).

# In[15]:

km = KMeans()
data = load_HTRU2()
_,labels = np.unique(data[1],return_inverse=True)
numClasses = len(np.unique(labels))
print('# of Classes: ',numClasses)
km.cluster(data[0],numClasses)
print('Mutual Information Score: ',km.normalized_mutual_information(data[0],data[1]))

#
kMeans did the worst job clustering according to class labels on the HTRU2 dataset, with a mutual information score of 0.025 . This tells us that there is almost no statistical relationship between clusters and class labels. This also means the clusters are not representative of class labels. It's worth noting that kMeans can get stuck in local optima, it's possible that the converged cluster centers are not global optima and we would need to impliment random restarts to help address this issue. The poor mutual information score most likely is caused by noisy features in the dataset, these features would have a large impact on cluster assignments (because of high distances) and no correlation to class labels.

# In[ ]:


def welcome():
  """ This will be welcome screen on the game"""
  print("""
 [H]=======ADAM-ASMACA======[-][o][x]
 |                                 |
 |          Hoşgeldiniz!           |
 |           Sürüm 0.1             |
 |    Devam etmek için kullanıcı   |
 |         adınızı girin.          |
 |                                 |
 |=================================|
""")

class User:
  """Base class for a user of game."""

  def __init__(self):
    """Constructor. Called when game started to create User attribute.
    (name, point, mistake, known_character_count, guested_characters)"""

    self.name = input("Choose your player name: ")
    self.point = 0
    self.mistake = 0
    self.mistakes_all = ["0", "/", "|", "\\", "/", "\\"]
    self.mistakes_first = ["","","","","",""]
    self.known_character_count = 0
    self.guested_characters = [] # This will be like ["a", "b", "c" .....]
    self.word_copy = []

  def print_user_name(self):
    print(self.name)

class Game:
  """Class for game objects.
  Word list - it will be sqlite database"""

  word_pool = ["iskorpit", "kalemlik", "bilgisayar"]

  def __init__(self, user):
    """Constructor. Called when game start"""

    self.word = self.word_pool[0].upper() # select random word in database
    self.word_length = len(self.word) # We will use this with total mistake
    user.word_copy = ["_" for i in range(self.word_length) ]

class Control:
  """Game Controller Class"""

  word_list = list("ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZ")

  def user_general_info(self, user, game):
    """This method print all user general info"""
    print("""
  user name: {}
  word length: {}
  User mistake: {}
  word status: {}
  guested characters: {}
  """.format(user.name, game.word_length, user.mistake,
  user.word_copy, user.guested_characters,))

    print("""
   _________
  |         |
  |         {}
  |        {}{}{}
  |        {} {}
  |
  |
 ----

    """.format(*user.mistakes_first))

  def is_mistake_full(self, user):  # 6 is limit for mistake
    if user.mistake > 5:
      print("Game Over, You Lose!")
      exit()
    else:
      pass

  def is_all_char_found(self, user, game):
    if user.known_character_count >= game.word_length:
      print("Congratulations You Won!")
      exit()
    else:
      pass

  def get_one_char(self, user, game):
    self.get_char = input("Bir harf giriniz: ").upper()

    if self.get_char == "":
      print("Boş giremezsiniz!")
      self.get_one_char(user, game)
    elif len(self.get_char) > 1:
      print("Tek harf girmelisiniz!")
      self.get_one_char(user, game)
    elif self.get_char not in self.word_list:
      print("Yalnızca latin harfleri girebilirsiniz!")
      self.get_one_char(user, game)
    elif self.get_char in user.guested_characters:
      print("Daha önce girilen harfleri giremezsiniz!")
      self.get_one_char(user, game)
    else:
      print("Kelime kontrol ediliyor...") # Letter passed

  def is_char_in_word(self, user, game):
    if self.get_char in game.word:
      print("Tebrikler bu harf var!")
      for i in range(len(game.word)):
        if self.get_char == game.word[i]:
          user.word_copy[i] = self.get_char
          user.known_character_count += 1
          self.is_all_char_found(user, game)
      print(user.word_copy)
    else:
      print("Üzgünüm '{}' harfi kelimede yok.".format(self.get_char))
      # user mistake list changing for hanging
      user.mistakes_first[user.mistake] = user.mistakes_all[user.mistake]
      user.mistake += 1
      self.is_mistake_full(user)

    user.guested_characters.append(self.get_char)

  def make_user_guest(self, user, game):
    self.user_guest = input("Tahminde bulunmak ister misiniz: E-H").upper()
    if self.user_guest == "E":
      self.guest = input("Tahmininizi yazınız: ").upper()
      if self.guest == game.word:
        print("Tebrikler kazandınız!")
        exit()
      else:
        user.mistakes_first[user.mistake] = user.mistakes_all[user.mistake]
        user.mistake += 1
        if user.mistake >= game.word_length:
          print("Oyunu kaybettiniz!")
          exit()
      print("Tahmin tutmadı harf tahmin etmeye devam!")
    else:
      print("Harf tahmin etmeye devam!")

def main():
  welcome()
  user = User()
  game = Game(user)
  control = Control()
  while True:
    control.user_general_info(user, game)
    control.is_mistake_full(user) # user.mistake control
    control.is_all_char_found(user, game) # all character is found?
    control.get_one_char(user, game)
    control.is_char_in_word(user, game)
    control.make_user_guest(user, game)

if __name__ == "__main__":
    main()


#Joshua Hsin ID# 13651420
import json
from json2html import *
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from urllib.request import urlopen
import ssl
import os
import re
import sys
import _pickle as pickle
##https://docs.python.org/3/library/pickle.html
from collections import defaultdict
from nltk.stem import PorterStemmer
#https://pythonprogramming.net/stemming-nltk-tutorial/
import warnings
import math

warnings.filterwarnings("ignore", category=UserWarning, module='bs4')
ps = PorterStemmer()
key = dict() #key dictionary with urls
word_dict = defaultdict(dict) #word_dictionary

drop = open("dropoff.txt", "wb")
drop2 = open("dropoff2.txt", "wb")
drop3 = open("dropoff3.txt", "wb")
drop_final_list = [open("a.txt", "wb"), open("b.txt", "wb"), open("c.txt", "wb"), open("d.txt", "wb"), open("e.txt", "wb"), open("f.txt", "wb"),
                   open("g.txt", "wb"), open("h.txt", "wb"), open("i.txt", "wb"), open("j.txt", "wb"), open("k.txt", "wb"), open("l.txt", "wb"),
                   open("m.txt", "wb"), open("n.txt", "wb"), open("o.txt", "wb"), open("p.txt", "wb"), open("q.txt", "wb"), open("r.txt", "wb"),
                   open("s.txt", "wb"), open("t.txt", "wb"), open("u.txt", "wb"), open("v.txt", "wb"), open("w.txt", "wb"), open("x.txt", "wb"),
                   open("y.txt", "wb"), open("z.txt", "wb"), open("extra.txt", "wb")]
stop_words = ['a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an','and', 'any', 'are','aren\'t', 'as'
              , 'at', 'be', 'because', 'been', 'before', 'being', 'below','between', 'both', 'but', 'by', 'can\'t',
              'cannot', 'could', 'couldn\'t', 'did', 'didn\'t', 'do', 'does', 'doesn\'t', 'doing', 'don\'t', 'down', 'during',
              'each', 'few', 'for', 'from', 'further', 'had', 'hadn\'t', 'has', 'hasn\'t', 'have', 'haven\'t', 'having', 'he',
              'he\'d', 'he\'ll', 'he\'s', 'her', 'here', 'here\'s', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'how\'s',
              'i', 'i\'d', 'i\'ll', 'i\'m', 'i\'ve', 'if', 'in', 'into', 'is', 'isn\'t', 'it', 'it\'s', 'its', 'itself', 'let\'s',
              'me', 'more', 'most', 'mustn\'t', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or',
              'other', 'ought', 'our', 'ours']
doc_num = 1

def drop_word(d, word, value):
    #dump a word dictionary into appropriate file
    global doc_num
    for x in d[word].keys():
        if(d[word][x] > 0):
            d[word][x] *= math.log(doc_num/len(d[word].keys()))
    pickle.dump(d, drop_final_list[value])

def merge1(drop, a):
    #merge the rest of one file
    global doc_num
    finish = False
    while not finish:
        try:
            a_word = list(a.keys())[0]
            for x in a[a_word].keys():
                a[a_word][x] *= math.log(doc_num/len(a[a_word].keys()))
            if(0 <= (ord(a_word[0]) - 97) <= 25):
                pickle.dump(a,drop_final_list[ord(a_word[0]) - 97])
            else:
                pickle.dump(a,drop_final_list[-1])
            a = pickle.load(drop)
        except EOFError:
            finish = True
    return

def merge2(drop, drop2, a, b):
    #merge two files together
    a_word = list(a.keys())[0]
    b_word = list(b.keys())[0]
    finish = False
    while not finish:
        if(a_word < b_word):
            try:
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(a,a_word, value)
                else:
                    drop_word(a,a_word, -1)
                a = pickle.load(drop)
                a_word = list(a.keys())[0]
            except EOFError:
                merge1(drop2, b)
                finish = True
        elif(a_word == b_word):
            d = defaultdict(dict)
            d[a_word] = {**a[a_word], **b[b_word]}
            value = ord(a_word[0]) - 97
            if(0 <= value <= 25):
                drop_word(d,a_word, value)
            else:
                drop_word(d,a_word, -1)
            try:
                a = pickle.load(drop)
                a_word = list(a.keys())[0]
                try:
                    b = pickle.load(drop)
                    b_word = list(b.keys())[0]
                except EOFError:
                    merge1(drop, a)
                    finish = True
            except EOFError:
                try:
                    b = pickle.load(drop)
                    merge1(drop2, b)
                    finish = True
                except EOFError:
                    finish = True
        elif(a_word > b_word):
            #print(a,b)
            try:
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(b,b_word, value)
                else:
                    drop_word(b,b_word, -1)
                b = pickle.load(drop2)
                b_word = list(b.keys())[0]
            except EOFError:
                merge1(drop,a)
                finish = True
    return

def merge():
    #merge all 3 files together
    drop = open('dropoff.txt', 'rb')
    drop2 = open('dropoff2.txt', 'rb')
    drop3 = open('dropoff3.txt', 'rb')
    a = pickle.load(drop)
    b = pickle.load(drop2)
    c = pickle.load(drop3)
    a_word = list(a.keys())[0]
    b_word = list(b.keys())[0]
    c_word = list(c.keys())[0]
    finish = False
    while not finish:
        #compare all three files to determine which has the lowest value word
        #dump the lowest value word into the appropriate file (a - z and extra)
        #if files have the same word, merge word dicts, then dump word_dict
        if(a_word < b_word):
            if(a_word < c_word):
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(a,a_word, value)
                else:
                    drop_word(a,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                except EOFError:
                    merge2(drop2, drop3, b, c)
                    finish = True
            elif(a_word == c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **c[c_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                    try:
                        c = pickle.load(drop3)
                        c_word = list(c.keys())[0]
                    except EOFError:
                        merge2(drop, drop2, a, b)
                        finish = True
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop2, drop3, b, c)
                        finish = True
                    except EOFError:
                        merge1(drop2, b)
                        finish = True
            elif(c_word < a_word):
                #print('3',a_word,b_word,c_word)
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
        elif(a_word == b_word):
            if(b_word < c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **b[b_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                    try:
                        b = pickle.load(drop2)
                        b_word = list(b.keys())[0]
                    except EOFError:
                        merge2(drop, drop3, a, c)
                        finish = True
                except EOFError:
                    try:
                        b = pickle.load(drop2)
                        merge2(drop2, drop3, b, c)
                        finish = True
                    except EOFError:
                        merge1(drop3, c)
                        finish = True
            elif(b_word == c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **b[b_word], **c[c_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    try:
                        b = pickle.load(drop2)
                        try:
                            c = pickle.load(drop3)
                        except EOFError:
                            merge2(drop, drop2, a, b)
                            finish = True
                    except EOFError:
                        try:
                            c = pickle.load(drop3)
                            merge2(drop, drop2, a, c)
                            finish = True
                        except EOFError:
                            merge1(drop,a)
                            finish = True
                except EOFError:
                    try:
                        b = pickle.load(drop2)
                        try:
                            c = pickle.load(drop3)
                            merge2(drop2, drop3, b, c)
                            finish = True
                        except EOFError:
                            merge1(drop3, b)
                            finish = True
                    except EOFError:
                        try:
                            c = pickle.load(drop3)
                            merge1(drop3, c)
                            finish = True
                        except EOFError:
                            finish = True
                a_word = list(a.keys())[0]
                b_word = list(b.keys())[0]
                c_word = list(c.keys())[0]
            elif(b_word > c_word):
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
        elif(b_word < a_word):
            if(b_word < c_word):
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(b,b_word, value)
                else:
                    drop_word(b,b_word, -1)
                try:
                    b = pickle.load(drop2)
                    b_word = list(b.keys())[0]
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop, drop3, a, c)
                        finish = True
                    except EOFError:
                        merge1(drop, a)
                        finish = True
            elif(b_word == c_word):
                d = defaultdict(dict)
                d[b_word] = {**b[b_word], **c[c_word]}
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,b_word, value)
                else:
                    drop_word(d,b_word, -1)
                try:
                    b = pickle.load(drop2)
                    b_word = list(b.keys())[0]
                    try:
                        c = pickle.load(drop3)
                        c_word = list(c.keys())[0]
                    except:
                        merge2(drop, drop2, a, b)
                        finish = True
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop, drop3, a, c)
                        finish = True
                    except EOFError:
                        merge1(drop, a)
                        finish = True
            elif(b_word > c_word):
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
    return

def drop_off3(word_dict):
    #drop off current word_dict into drop3 file
    global drop3
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop3)
    return

def drop_off2(word_dict):
    #drop off current word_dict into drop2 file
    global drop2
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop2)
    return

def drop_off(word_dict):
    #drop off current word_dict into drop file
    global drop
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop)
    return

def short_word(word):
    #if word is <= 1 character, not a or i, and not a number return empty word
    if(len(word) <= 1 and (word != 'a' and word != 'i' and word != 'A' and word != 'I' and word not in ['0','1','2','3','4','5','6','7','8','9'])):
        word = ''
    elif(len(word) > 20):
        word = ''
    return word

def check(word):
    #check word
    if(re.match(r"([.]{0})([\-]{1,})([.]{0})$", word) or re.match(r"(.)*(https://)(.)*", word) or re.match(r"(.)*(http://)(.)*", word)):
        #get rid of words with multiple dashes and urls
        return ''
    if(re.match(r"([0-9]{1,2})(:{1})([0-5]{1}[0-9]{1})(am|pm)$", word)):
        return ''
    if(re.match(r"([0-9]{1,2})(st|nd|rd|th{1})$", word)):
        return word
    if(re.match(r"(.)*(.edu)$", word) and word != 'uci.edu' and word != 'ics.edu' and word != 'ics.uci.edu'):
        #return emails as is
        return word
    if(re.match(r"(([A-Za-z]{1})(\.{1})){2,}", word)):
        #Get rid of extraneous parts of words if acronym in word, then stem
        word = re.sub(r"[^a-z|^A-Z]", '', word)
        return word
    word = re.sub(r"[^a-z|^A-Z|^\'|^\-]", ' ', word)
    if(re.match(r"([\']{3})([A-Za-z\-]+)([\']{3})$", word)):
        #if quotations around word, get rid of them
        word = word[3:-3]
        word = ps.stem(word)
        return word
    if(re.match(r"([A-Za-z\-]+)([\']{3})$", word)):
        #if quotations after word, get rid of them
        word = word[:-3]
        word = ps.stem(word)
        return word
    if(re.match(r"([\']{3})([A-Za-z\-]+)$", word)):
        #if quotations before word, get rid of them
        word = word[3:]
        word = ps.stem(word)
        return word
    if(re.match(r"([\']{1})([A-Za-z\-]+)([\']{1})$", word) or re.match(r"([\-]{1})([A-Za-z\']+)([\-]{1})$",word)):
        #if quotations or dashes around word, get rid of them
        word = word[1:-1]
        word = ps.stem(word)
        return word
    if(re.match(r"([A-Za-z\-]+)([\']{1})$", word) or re.match(r"([A-Za-z\']+)([\-]{1})$", word)):
        #if quotations or dashes after word, get rid of them
        word = word[:-1]
        word = ps.stem(word)
        #word = "".join(word.split())
        return word
    if(re.match(r"([\']{1})([A-Za-z\-]+)$", word) or re.match(r"([\-]{1})([A-Za-z\']+)$", word)):
        #if quotations or dashes before word, get rid of them
        word = word[1:]
        word = ps.stem(word)
        return word
    if(word[-2] != "\'"):
        word = ps.stem(word)
    return word

def insert(word, value):
    #insert words and values into word_dict
    #if word already exists, increment value of appropriate document for word
    global word_dict
    global doc_num
    if word != '':
        if ' ' in word:
            words = word.split()
            for i in words:
                i = check(i)
                if i != '':
                    if(len(i) != 1 or i in ['0','1','2','3','4','5','6','7','8','9']):
                        if i not in word_dict.keys():
                            word_dict[i][doc_num] = value
                        else:
                            if(doc_num not in word_dict[i].keys()):
                                word_dict[i][doc_num] = value
                            else:
                                word_dict[i][doc_num] += value
        else:
            if(word not in word_dict.keys()):
                word_dict[word][doc_num] = value
            else:
                if(doc_num not in word_dict[word].keys()):
                    word_dict[word][doc_num] = value
                else:
                    word_dict[word][doc_num] += value
    return word_dict

def parse(soup, tag, value):
    #parse through the json file words
    global word_dict
    a = soup.find_all(tag)
    for x in range(len(a)):
        i = a[x].text
        for word2 in i.split():
            word = check(word2.lower())
            word_dict = insert(word, value)
    return word_dict

def index():
    #index words into inverted index
    global key
    global doc_num
    global word_dict
    directory_num = 1

    ssl._create_default_https_context = ssl._create_unverified_context
    #https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error
    for subdir, dirs, files in os.walk('DEV'):
        #https://stackoverflow.com/questions/19587118/iterating-through-directories-with-python
        print(str(subdir))
        for filename in files:
            if(str(os.path.join(subdir, filename)).endswith('.DS_Store')):
                #ignore extraneous files (for Mac)
                pass
            else:
                #load files, put url into key dictionary
                a = open(str(os.path.join(subdir, filename)), 'r')
                b = json.load(a)
                key[doc_num] = str(b['url'])
                #https://pypi.org/project/json2html/
                try:
                    #find tags with words and index
                    soup = BeautifulSoup(b['content'], 'lxml')
                    #https://www.crummy.com/software/BeautifulSoup/bs4/doc/
                    if(soup.find('title') != None):
                        word_dict = parse(soup, 'title', 1.5)
                    if(soup.find('p') != None):
                        word_dict = parse(soup, 'p', 1)
                    if(soup.find('h1') != None):
                        word_dict = parse(soup, 'h1', 1.5)
                    if(soup.find('h2') != None):
                        word_dict = parse(soup, 'h2', 1.5)
                    if(soup.find('h3') != None):
                        word_dict = parse(soup, 'h3', 1.5)
                    if(soup.find('h4') != None):
                        word_dict = parse(soup, 'h4', 1)
                    if(soup.find('h5') != None):
                        word_dict = parse(soup, 'h5', 1)
                    if(soup.find('h6') != None):
                        word_dict = parse(soup, 'h6', 1)
                    if(soup.find('b') != None):
                        word_dict = parse(soup, 'b', 1.5)
                    if(soup.find('strong') != None):
                        word_dict = parse(soup, 'strong', 1.5)
                    if(soup.find('i') != None):
                        word_dict = parse(soup, 'i', 1.5)
                    if(soup.find('u') != None):
                        word_dict = parse(soup, 'u', 1.5)
                    doc_num += 1
                except:
                    pass
        if(directory_num == 29):
            drop_off(word_dict)
            global drop
            drop.close()
            word_dict.clear()
        elif(directory_num == 59):
            drop_off2(word_dict)
            global drop2
            drop2.close()
            word_dict.clear()
        elif(directory_num == 89):
            drop_off3(word_dict)
            global drop3
            drop3.close()
            merge()
            break
        directory_num += 1
    #close files
    drop.close()
    drop2.close()
    drop3.close()
    for x in range(len(drop_final_list)):
        drop_final_list[x].close()

    for x in range(0, 26):
        drop_final_list[x] = open(chr(x + 97) + '.txt', 'rb')
    drop_final_list[-1] = open('extra.txt', 'rb')

if __name__ == '__main__':
    index()
    print('done')


from many_values import ManyValues
from single_value import SingleValue

single_value = SingleValue(11)
other_values = ManyValues()
other_values.append(22)
other_values.append(33)
# make a list of all values
all_values = ManyValues()
all_values.append(single_value)
all_values.append(other_values)
print(all_values.sum)


# tests.py

from unittest import TestCase, main as unittest_main
from app import app

class PlaylistsTests(TestCase):
    """Flask tests."""

    def setUp(self):
        """Stuff to do before every test."""

        # Get the Flask test client
        self.client = app.test_client()

        # Show Flask errors that happen during tests
        app.config['TESTING'] = True

#     # tests.py
# class PlaylistsTests(TestCase):
#     ...
#     def test_index(self):
#         """Test the playlists homepage."""
#         result = self.client.get('/')
#         self.assertEqual(result.status, '200 OK')

class PlaylistsTests(TestCase):

    def test_new(self):
        """Test the new playlist creation page."""
        result = self.client.get('/playlists/new')
        self.assertEqual(result.status, '200 OK')

    @mock.patch('pymongo.collection.Collection.delete_one')
    def test_delete_playlist(self, mock_delete):
        form_data = {'_method': 'DELETE'}
        result = self.client.post(f'/playlists/{sample_playlist_id}/delete', data=form_data)
        self.assertEqual(result.status, '302 FOUND')
        mock_delete.assert_called_with({'_id': sample_playlist_id})

if __name__ == '__main__':
    unittest_main()


import yaml
import pprint
import sys

def test_sort_keys(input_filename, sorted_filename, verbose=False):
    with open(input_filename, 'rb') as file:
        input = file.read().decode('utf-8')
    with open(sorted_filename, 'rb') as file:
        sorted = file.read().decode('utf-8')
    data = yaml.load(input, Loader=yaml.FullLoader)
    dump_sorted = yaml.dump(data, default_flow_style=False, sort_keys=True)
    dump_unsorted = yaml.dump(data, default_flow_style=False, sort_keys=False)
    dump_unsorted_safe = yaml.dump(data, default_flow_style=False, sort_keys=False, Dumper=yaml.SafeDumper)
    if verbose:
        print("INPUT:")
        print(input)
        print("DATA:")
        print(data)

    assert dump_sorted == sorted

    if sys.version_info>=(3,7):
        assert dump_unsorted == input
        assert dump_unsorted_safe == input

test_sort_keys.unittest = ['.sort', '.sorted']

if __name__ == '__main__':
    import test_appliance
    test_appliance.run(globals())


# change the categorical variables to dummies
def from_categorical_to_dummies(CV_data, var):
    import pandas as pd
    import numpy as np

    df = pd.get_dummies(CV_data[var])
    ## eliminate the variable with minimum variance
    l = df.std()
    #l.idxmin() #kjkrfgld_UhSVl
    df = df.drop(columns = l.idxmin())
    #columns and tranformation
    col = list(df.columns)
    names = np.repeat(var, len(col), axis=0)
    col = paste(names, col, sep = '_')
    df.columns = col

    # Join the dummy variables to the main dataframe
    df_new = pd.concat([CV_data, df], axis=1)
    df_new[col] = df_new[col].apply(pd.to_numeric, downcast='float')

    CV_data = df_new
    CV_data = CV_data.drop(columns = var)
    return(CV_data)

# paste function: used to concatenate any type of variables
import functools
def reduce_concat(x, sep=""):
    return functools.reduce(lambda x, y: str(x) + sep + str(y), x)

def paste(*lists, sep=" ", collapse=None):
    result = map(lambda x: reduce_concat(x, sep=sep), zip(*lists))
    if collapse is not None:
        return reduce_concat(result, sep=collapse)
    return list(result)

# group variables by target
def group_var_target(dataset, target, var):
    g = dataset.groupby(var)
    x = g[target].value_counts(normalize=True)
    x = x.reset_index(var)
    x.columns = [var, 'count']
    x = x.reset_index(target)
    return x

# fill numeric na's
def fill_numeric_nas(dataset, k):
    #conda install -c brittainhard fancyimpute
    import numpy as np
    import pandas as pd
    from fancyimpute import KNN, MICE
    dataset['poor'] = dataset['poor'].astype('str')
    df_numeric = dataset.select_dtypes(include=[np.float])
    df_int = dataset.select_dtypes(include=[np.uint8])
    df_str = dataset.select_dtypes(include=[np.object])
    col = df_numeric.columns
    df_filled = pd.DataFrame(KNN(k, verbose=False).complete(df_numeric))
    #df_filled = pd.DataFrame(MICE(n_imputations=200, impute_type='col', verbose=False).complete(df_numeric.as_matrix()))
    df_filled.columns = col
    df = pd.concat([df_filled, df_str], axis=1)
    df = pd.concat([df, df_int], axis=1)
    return df

# correlations

# result can be used w/ seaborn's heatmap
def compute_correlation_matrix(df, method='pearson'):
    import pandas as pd
    corr_mat = df.corr()
    return corr_mat

def prepare_data(dataset, coll):
   import numpy as np
   # MANAGE OUTLIERS
   dataset = fix_extreme_values(dataset)
   # ELIMINATE VARIABLES To avoid multicolineality
   #
   # insert values in categorical variables
   dataset.kjkrfgld.fillna('MISSING',inplace = True)
   dataset.bpowgknt.fillna('MISSING',inplace = True)
   dataset.raksnhjf.fillna('MISSING',inplace = True)
   dataset.vwpsxrgk.fillna('MISSING',inplace = True)
   #fill numeric na's
   dataset = fill_numeric_nas(dataset, 3)
   # include interactions
   dataset['inter_1'] = dataset['yfmzwkru'] * dataset['omtioxzz']
   dataset['inter_2'] = dataset['yfmzwkru'] * dataset['tiwrsloh']
   dataset['inter_3'] = dataset['yfmzwkru'] * dataset['weioazcf']
   dataset['inter_4'] = dataset['omtioxzz'] * dataset['tiwrsloh']
   dataset['inter_5'] = dataset['omtioxzz'] * dataset['weioazcf']
   dataset['inter_6'] = dataset['tiwrsloh'] * dataset['weioazcf']
   # one-hot-encodign
   dataset_complete = from_categorical_to_dummies(dataset, 'kjkrfgld')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'bpowgknt')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'raksnhjf')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'vwpsxrgk')

   cols = ['omtioxzz', 'tiwrsloh', 'weioazcf', 'yfmzwkru',  'inter_3',
'inter_4', 'inter_6','kjkrfgld_GVYCk', 'kjkrfgld_MISSING', 'kjkrfgld_qzGkS', 'kjkrfgld_tnDpM', 'bpowgknt_MISSING', 'raksnhjf_qTmDg', 'raksnhjf_rXCdD', 'vwpsxrgk_ARuYG', 'vwpsxrgk_MISSING', 'vwpsxrgk_QRKWz',  'vwpsxrgk_bUhyU', 'vwpsxrgk_puHHm', 'vwpsxrgk_qcizG', 'vwpsxrgk_yKWYC'] #'omtioxzz', 'tiwrsloh', 'weioazcf',

  # filter variables
   if len(coll) <= 1:
      dataset_complete = dataset_complete.drop(cols, axis=1)
   else:
      dataset_complete = dataset_complete.loc[:, coll]
      dataset_complete = dataset_complete.fillna(0)

   d = {'False' : '0', 'True' : '1'}
   dataset_complete['poor'] = dataset_complete['poor'].map(d)
   dataset_complete['poor'] = dataset_complete['poor'].astype('int')
   return dataset_complete

def fix_extreme_values(df_numeric):
   # Fix the extreme values
   x = df_numeric['yfmzwkru'].nlargest(2)
   x = pd.DataFrame(x)
   out_max = x.max()
   next_value = x.min()
   df_numeric.loc[df_numeric['yfmzwkru'] >= out_max[0], 'yfmzwkru'] = next_value[0]

   x = df_numeric['omtioxzz'].nsmallest(2)
   x = pd.DataFrame(x)
   out_min = x.min()
   next_value = x.max()
   df_numeric.loc[df_numeric['omtioxzz'] == out_min[0], 'omtioxzz'] = next_value[0]
   return df_numeric

import matplotlib.pyplot as plt
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix'):

    import matplotlib.pyplot as plt
    import numpy as np
    import itertools
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap= plt.cm.Blues ) #, cmap=cmap
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

######################## -------------------   CHI-SQUARE --------------------------------------------------------------

import pandas as pd
import numpy as np
import scipy.stats as stats
from scipy.stats import chi2_contingency

class ChiSquare:
    def __init__(self, dataframe):
        self.df = dataframe
        self.p = None #P-Value
        self.chi2 = None #Chi Test Statistic
        self.dof = None

        self.dfObserved = None
        self.dfExpected = None

    def _print_chisquare_result(self, colX, alpha):
        result = ""
        if self.p<alpha:
            result="{0} is IMPORTANT for Prediction".format(colX)
        else:
            result="{0} is NOT an important predictor. (Discard {0} from model)".format(colX)

        print(result)

    def TestIndependence(self,colX,colY, alpha=0.05):
        X = self.df[colX].astype(str)
        Y = self.df[colY].astype(str)

        self.dfObserved = pd.crosstab(Y,X)
        chi2, p, dof, expected = stats.chi2_contingency(self.dfObserved.values)
        self.p = p
        self.chi2 = chi2
        self.dof = dof

        self.dfExpected = pd.DataFrame(expected, columns=self.dfObserved.columns, index = self.dfObserved.index)

        self._print_chisquare_result(colX,alpha)


'''
Log Report Exporter for Cloudwatch log group to S3
Report's are saved to bucket
'''

import boto3
import json
import logging
import os
import time
import csv

from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
from datetime import datetime, timedelta

logger = logging.getLogger()
logger.setLevel(logging.INFO)

# The Slack channel to send a message to stored in the slackChannel environment variable
SLACK_CHANNEL = os.environ['slackChannel']
HOOK_URL = os.environ['HOOK_URL']

# cloudwatch log insights
client = boto3.client('logs')
log_group = ''

# S3 bucket for report
s3 = boto3.client('s3')
bucket_name = os.environ['bucket_name']
bucket_path = os.environ['bucket_path'] if os.environ['bucket_path'] else ''

# time
now = datetime.now()
year = now.year
month = now.month
day = now.day
hour = now.hour
minute = now.minute

# create task to export cloudwatch log group
def create_task():
    endTime = datetime.now()
    startTime = endTime - timedelta(minutes=10)
    task_id = client.create_export_task(
        logGroupName=log_group,
        destination=f"{bucket_name}",
        fromTime=int(startTime.timestamp())*1000,       # from/to timestamp -> millisecond format
        to=int(endTime.timestamp())*1000,
        destinationPrefix=f"{bucket_path}/{year}/{month}/{day}"
        )
    return task_id['taskId']

# get task result with task id
def get_task_result(task_id):
    retry = 0
    sleep_time, max_retry=2, 5
    status = 'RUNNING'
    while status == 'RUNNING' and retry < max_retry:
        time.sleep(sleep_time)
        response = client.describe_export_tasks(taskId=task_id)
        status = response['exportTasks'][0]['status']['code']
        logger.info(f"TASK STATUS : {status}")
        retry += 1
    if status == 'COMPLETED':
        logger.info(f"Task {task_id} is completed")
        return status

def lambda_handler(event, context):
    task_id = create_task()
    status = get_task_result(task_id)

    if status == 'COMPLETED':
        if hour == 0 and minute == 0:
            y = now - timedelta(days=1)
            obj_list = s3.list_objects_v2(
                Bucket=bucket_name,
                Prefix=f"{bucket_path}/{y.year}/{y.month}/{y.day}"
            )

            logger.info(f"{y.year}/{y.month}/{y.day} Summary")
            logger.info(f"{len(obj_list['Contents'])} reports saved")
            slack_message = {
                'channel': SLACK_CHANNEL,
                'text': f"*[WAF-log-exporter]*\t{len(obj_list['Contents'])} task has completed"
            }
            req = Request(HOOK_URL, json.dumps(slack_message).encode('utf-8'))
            try:
                response = urlopen(req)
                response.read()
                logger.info("Message posted to %s", slack_message['channel'])
            except HTTPError as e:
                logger.error("Request failed: %d %s", e.code, e.reason)
            except URLError as e:
                logger.error("Server connection failed: %s", e.reason)
        else:
            logger.info(f"Task {task_id} is completed")
    else:
        logger.error(f"Task {task_id} is failed")


#whole framework should be scirptable as well as runnable with start_fby script
#or something like that
import os
import time
import datetime
from fbpy.environment import JSONFile, FlowBasedEnvironment

THIS_DIR = os.path.realpath(os.path.dirname(__file__))

if __name__=="__main__":
    config = JSONFile(os.path.join(THIS_DIR, "config.json"))
    net = JSONFile(os.path.join(THIS_DIR, "net.json"))
    runtime = JSONFile(os.path.join(THIS_DIR, "runtime.json"))

    env = FlowBasedEnvironment(config, net, runtime)
    env.set_up()
    env.up()
    print("started at", datetime.datetime.utcnow())
    time.sleep(60)
    print("sleep finished at", datetime.datetime.utcnow())
    env.down()


# -*- coding: utf-8 -*-

import re
from .schema_manager import SchemaManager
from .platforms.sqlite_platform import SQLitePlatform
from .column import Column

class SQLiteSchemaManager(SchemaManager):

    def _get_portable_table_column_definition(self, table_column):
        parts = table_column['type'].split('(')
        table_column['type'] = parts[0]
        if len(parts) > 1:
            length = parts[1].strip(')')
            table_column['length'] = length

        db_type = table_column['type'].lower()
        length = table_column.get('length', None)
        unsigned = False

        if ' unsigned' in db_type:
            db_type = db_type.replace(' unsigned', '')
            unsigned = True

        fixed = False

        type = self._platform.get_type_mapping(db_type)
        default = table_column['dflt_value']
        if default == 'NULL':
            default = None

        if default is not None:
            # SQLite returns strings wrapped in single quotes, so we need to strip them
            default = re.sub("^'(.*)'$", '\\1', default)

        notnull = bool(table_column['notnull'])

        if 'name' not in table_column:
            table_column['name'] = ''

        precision = None
        scale = None

        if db_type in ['char']:
            fixed = True
        elif db_type in ['float', 'double', 'real', 'decimal', 'numeric']:
            if 'length' in table_column:
                if ',' not in table_column['length']:
                    table_column['length'] += ',0'

                precision, scale = tuple(map(lambda x: x.strip(), table_column['length'].split(',')))

            length = None

        options = {
            'length': length,
            'unsigned': bool(unsigned),
            'fixed': fixed,
            'notnull': notnull,
            'default': default,
            'precision': precision,
            'scale': scale,
            'autoincrement': False
        }

        column = Column(table_column['name'], type, options)
        column.set_platform_option('pk', table_column['pk'])

        return column

    def list_table_indexes(self, table):
        sql = self._platform.get_list_table_indexes_sql(table)

        cursor = self._connection.get_connection().cursor()
        table_indexes = cursor.execute(sql).fetchall()

        indexes = []
        for index in table_indexes:
            table_index = dict(index.items())
            index_info = cursor.execute('PRAGMA index_info(%s)' % index['name']).fetchall()
            columns = []
            for column in index_info:
                columns.append(column['name'])

            table_index['columns'] = columns

            indexes.append(table_index)

        return indexes

    def get_database_platform(self):
        return SQLitePlatform()


from time import sleep

from selenium import webdriver

class Baidu_Search():

    def __init__(self):
        self.dev = webdriver.Chrome()
        self.dev.get("https://www.baidu.com")
        self.dev.implicitly_wait(10)

    def search(self):
        self.dev.find_element_by_xpath("//input[@id='kw']").send_keys("python")
        self.dev.find_element_by_xpath("//input[@value='百度一下']").click()
        sleep(2)

    def out(self):
        self.dev.quit()

if __name__ == '__main__':
    obj = Baidu_Search()
    obj.search()
    obj.out()


import json, sqlite3

def get_department_name(id, cur):
    result = cur.execute("SELECT department_id FROM artwork__department WHERE artwork_id = \""+id+"\"").fetchone()

    department_id = result[0]

    result = cur.execute("SELECT name FROM department WHERE id = \""+department_id+"\"").fetchone()

    return result[0]

def get_creator_info(id, cur):
    result = cur.execute("SELECT creator_id FROM artwork__creator WHERE artwork_id=\""+id+"\"").fetchall()
    # if no creator we have to check
    if len(result) == 0:
        return []
    else:
        #some cases there is more than one creator
        creators_roles = list()
        for r in result:
            result = cur.execute("SELECT role, description FROM creator WHERE id = \""+r[0]+"\"").fetchone()
            if result:
                n = '{"role": "'+result[0].replace('"','\\"')+'", "description":"'+result[1].replace("'","’").replace('"','”')+'"}'
                creators_roles.append(json.loads(n))
        return creators_roles

def convert(input, output, department_file):
    # making connection with the sqlite file
    connection = sqlite3.connect(input)
    cur = connection.cursor()

    # "w+" means if not existed create one and if Overwrites the existing file
    f = open(output, "w+", encoding='utf-8')
    f.close()

    # "a" means that we will keep adding to the existing data every time we call f.write("some text")
    f = open(output, "a", encoding='utf-8')
    #start of array
    f.write("[")

    result = cur.execute("SELECT * FROM artwork").fetchall()

    # list(array) which will contain the json objects
    rows = list()
    for row in result:

        id = row[0].replace('"','\\"')
        accession_number = row[1].replace('"','\\"')
        title = row[2].replace('"','\\"')
        tombstone = row[3].replace('"','\\"')
        department = get_department_name(id, cur)
        creators = str(get_creator_info(id, cur)).replace("'",'"')

        rows.append('{"id":"'+id+'", "accession_number": "'+accession_number+'", "title": "'+title+'", "tombstone": "'+tombstone+'", "department": "'+department+'", "creators": '+creators+'}')

    #writing to the json file and sperate them with "," and  "\n" ( not be in the same line).
    f.write(",\n".join(rows))

    #end of the array
    f.write("]")

    f.close()
    # department file

    f = open(department_file, "w+", encoding='utf-8')
    f.close()

    f = open(department_file, "a", encoding='utf-8')
    t = list()
    result = cur.execute("SELECT * FROM department").fetchall()
    for r in result:
        t.append(r[1])
    f.write(str(t).replace("'",'"'))
    f.close()

#creating two JSON files
if __name__ == "__main__":
    sqlite3_file = "./cma-artworks.db"
    json_file = "./example.json"
    department_file = "./departments.json"
    convert(sqlite3_file, json_file, department_file)


"""The 'useragent' module contains useragent tools."""

__all__ = 'gen_useragents_cycle',

from itertools import cycle as _cycle
from typing import Iterator as _Iterator

def gen_useragents_cycle(file: str = None) -> _Iterator[str]:
    """Return itertools.cycle(`file` or 'Chrome 106.0 Win10')."""
    if file is None:
        return _cycle([
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, например Gecko) "
            "Chrome/106.0.0.0 Safari/537.36"])
    with open(file) as useragents:
        return _cycle(useragents.readlines())


from spack import *

class ArgpStandalone(AutotoolsPackage):
    """Standalone version of the argp interface from glibc for parsing
       unix-style arguments. """

    homepage = "https://www.lysator.liu.se/~nisse/misc"
    url      = "https://www.lysator.liu.se/~nisse/misc/argp-standalone-1.3.tar.gz"

    version('1.3', '720704bac078d067111b32444e24ba69')

    # Homebrew (https://github.com/Homebrew/homebrew-core) patches
    # argp-standalone to work on Darwin; the patchfile below was taken
    # from
    # https://raw.githubusercontent.com/Homebrew/formula-patches/b5f0ad3/argp-standalone/patch-argp-fmtstream.h
    patch('argp-fmtstream.h.patch', 0, 'platform=darwin', '.')

    def install(self, spec, prefix):
        make('install')
        make('check')
        mkdirp(self.spec.prefix.lib)
        install('libargp.a', join_path(self.spec.prefix.lib, 'libargp.a'))
        mkdirp(self.spec.prefix.include)
        install('argp.h', join_path(self.spec.prefix.include, 'argp.h'))


import argparse
import os
import numpy as np

print('''
Usage: convert_mat_2_ua --ua="ua root path"
''')

parser = argparse.ArgumentParser(description='UA Result Formatter')
parser.add_argument('--mot_folder', default=r"D:\ssj\DETRAC\20170721Result\0812-E25-M80-G30-TestSet-EB-MOT",
                    help='''mot result folder, with the following directory structure:
                    folder
                    |
                    |-- 0.1
                    |-- 0.2
                    |-- ...
                    ''')
parser.add_argument('--ua_folder', default=r"D:\ssj\DETRAC\20170721Result\0812-E25-M80-G30-TestSet-EB-UA", help='ua result folder. This tool would create this folder with same sturcture')

args = parser.parse_args()

class ConvertTools:
    @staticmethod
    def init(mot_folder, ua_folder):
        if not os.path.exists(mot_folder):
            raise FileNotFoundError('cannot find {}'.format(mot_folder))

        if not os.path.exists(ua_folder):
            os.mkdir(ua_folder)

        # get all the videos' frame number from 0.0 folder
        frame_number = {}
        path00 = os.path.join(mot_folder, '0.0')
        files00 = os.listdir(path00)
        for f in files00:
            if 'speed' not in f:
                frame_number[os.path.splitext(f)[0]] = max(np.loadtxt(os.path.join(path00, f), dtype=int)[:, 0])

        # get all directory in mot_folder path
        threshold_folder = [os.path.join(mot_folder, d) for d in os.listdir(mot_folder)]
        threshold_folder = list(filter(lambda f: os.path.isdir(f), threshold_folder))
        for folder in threshold_folder:
            ua_threshold_folder = os.path.join(ua_folder, os.path.basename(folder))
            if not os.path.exists(ua_threshold_folder):
                os.mkdir(ua_threshold_folder)
            # list all the files
            files = [os.path.join(folder, f) for f in os.listdir(folder)]
            files = sorted(list(filter(lambda f: os.path.isfile(f) and 'speed' not in os.path.basename(f), files)))
            for file in files:
                print('process: {}====>'.format(file))
                ua_file = os.path.join(ua_threshold_folder, os.path.splitext(os.path.basename(file))[0]) + "_{}.txt"
                data = np.loadtxt(file, dtype=int)
                if len(data) == 0:
                    np.savetxt(ua_file.format('LX'), [], fmt='%i')
                    np.savetxt(ua_file.format('LY'), [], fmt='%i', )
                    np.savetxt(ua_file.format('W'), [], fmt='%i')
                    np.savetxt(ua_file.format('H'), [], fmt='%i')
                    np.savetxt(ua_file.format('speed'), [], fmt='%f')
                    continue
                if len(data.shape) == 1:
                    data = np.expand_dims(data, axis=0)

                data[:, 0] = data[:, 0] - 1
                data[:, 1] = data[:, 1] - 1
                # max_f = max(data[:, 0])+1
                max_f = frame_number[os.path.splitext(os.path.basename(file))[0]]
                time = np.loadtxt(os.path.splitext(file)[0]+'-speed.txt', dtype=float)
                if time == 0:
                    speed = 0
                else:
                    speed = max_f / time
                max_id = max(data[:, 1])+1
                ua_data_LX = np.zeros((max_f, max_id), dtype=int)
                ua_data_LY = np.zeros((max_f, max_id), dtype=int)
                ua_data_H = np.zeros((max_f, max_id), dtype=int)
                ua_data_W = np.zeros((max_f, max_id), dtype=int)

                for row in data:
                    r = row[0]
                    c = row[1]
                    ua_data_LX[r, c] = row[2]
                    ua_data_LY[r, c] = row[3]
                    ua_data_W[r, c] = row[4]
                    ua_data_H[r, c] = row[5]
                # if len(data) > 1:
                #     ua_data_LX[0, :] = ua_data_LX[1, :]
                #     ua_data_LY[0, :] = ua_data_LY[1, :]
                #     ua_data_W[0, :] = ua_data_W[1, :]
                #     ua_data_H[0, :] = ua_data_H[1, :]

                np.savetxt(ua_file.format('LX'), ua_data_LX, fmt='%i')
                np.savetxt(ua_file.format('LY'), ua_data_LY, fmt='%i', )
                np.savetxt(ua_file.format('W'), ua_data_W, fmt='%i')
                np.savetxt(ua_file.format('H'), ua_data_H, fmt='%i')
                np.savetxt(ua_file.format('speed'), [speed], fmt='%f')

        # save sequence name file
        sequenceNames = [os.path.splitext(os.path.basename(f))[0] for f in os.listdir(threshold_folder[0])]
        np.savetxt(os.path.join(ua_folder, 'sequences.txt'), sequenceNames, fmt='%s', delimiter='\n')

        # save threshold file name
        threshold = os.listdir(mot_folder)
        np.savetxt(os.path.join(ua_folder, 'thresh.txt'), threshold, fmt='%s',delimiter='\n')

if __name__ == '__main__':
    # condition
    ConvertTools.init(args.mot_folder, args.ua_folder)


from fastapi import FastAPI
import uvicorn
from dotenv import dotenv_values
from pymongo import MongoClient
from routes.api import router as api_router
config = dotenv_values(".env")

app = FastAPI()

@app.on_event("startup")
def startup_db_client():
    app.mongodb_client = MongoClient(config["ATLAS_URI"])
    app.database = app.mongodb_client[config["DB_NAME"]]
    print("Project connected to the MongoDB database!")

@app.on_event("shutdown")
def shutdown_db_client():
    app.mongodb_client.close()

app.include_router(api_router)

if __name__ == '__main__': #this indicates that this a script to be run
    uvicorn.run("main:app", host='127.0.0.1', port=8000, log_level="info", reload = True)


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

from functools import reduce

def add(x, y):
    return x*10+y

sum = reduce(add, [1, 3, 5, 7, 9])

print(sum)
print('------------')

def prod(L):
   def fn(x, y):
      return x * y
   return reduce(fn, L)

print('3 * 5 * 7 * 9 =', prod([3, 5, 7, 9]))
if prod([3, 5, 7, 9]) == 945:
    print('测试成功!')
else:
    print('测试失败!')

print('---------')

def str2float(s):
    def fn(x,y):
        return x*10+y
    n=s.index('.')
    s1=list(map(int,[x for x in s[:n]]))
    s2=list(map(int,[x for x in s[n+1:]]))
    print(s1)
    print(s2)
    return reduce(fn,s1) + reduce(fn,s2)/10**len(s2)

print('str2float(\'123.456\') =', str2float('123.456'))
if abs(str2float('123.456') - 123.456) < 0.00001:
    print('测试成功!')
else:
    print('测试失败!')


import argparse
from textrank4zh import TextRank4Sentence

parser = argparse.ArgumentParser()
parser.add_argument("--dataset_folder_path",default='preproess_data/example_dataset',type=str)
parser.add_argument("--result_folder_path",default='./output',type=str)
parser.add_argument("--result_file_name",default='result.txt',type=str)
parser.add_argument("--selected_num",default=3,type=int)

args = parser.parse_args()
arg_dict=args.__dict__

#计算需要选择几句话
if arg_dict['selected_num']>0:
    selected_num=arg_dict['selected_num']
else:
    train_results=open(arg_dict['dataset_folder_path']+'/train.tgt').readlines()
    average_num=0
    for line in train_results:
        average_num+=len(line.split('</sentence>'))-1
    average_num/=len(train_results)
    selected_num=max(round(average_num),1)  #至少要选1句

#通过textrank4zh包计算得到结果
original_texts=open(arg_dict['dataset_folder_path']+'/test.src').readlines()
result_file=open(arg_dict['result_folder_path']+'/'+arg_dict['result_file_name'],'w',
                encoding='utf-8')
for d in original_texts:
    tr4s = TextRank4Sentence()
    sentence_list=d.split('</sentence>')[:-1]
    text='\n'.join(sentence_list)
    tr4s.analyze(text=text, lower=True, source = 'all_filters')
    pl=[]
    for item in tr4s.get_key_sentences(num=selected_num):
        pl.append(item.index)
    pl.sort()
    result=''.join([sentence_list[x] for x in pl])
    result_file.write(result+'\n')
result_file.close()


# Title: Ames Housing Statistical Analysis
# Author: Alexander Zakrzeski
# Date: January 17, 2023

# Import the necessary libraries and modules

import numpy as np
import pandas as pd
import scipy.stats as stats
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import variance_inflation_factor

import matplotlib.pyplot as plt
import seaborn as sns
import plotnine as pn

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LinearRegression, Lasso, LassoCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error

import janitor as jan
import movecolumn as mc

##### Part A - Data Preprocessing #####

# Take the following steps:
  # 1. Create a list
  # 2. Import the data
  # 3. Consolidate the column names
  # 4. Rename a specific column
  # 5. Create new columns
  # 6. Modifying existing columns
  # 7. Drop columns

cols1_drop = ["pid", "ms_subclass", "year_built", "year_remod_add",
              "fireplaces", "garage_yr_blt", "mo_sold", "yr_sold"]

data1_all = (
    jan.clean_names(pd.read_csv("Ames_Data.csv"))
       .rename(columns = {"order": "id"})
       .assign(age = lambda x: x["yr_sold"] - x["year_built"])
       .assign(age = lambda x: np.where(x["age"] < 0, 0, x["age"]),
               overall_cond = lambda x:
                   np.where(x["overall_cond"] < 5, "below",
                   np.where(x["overall_cond"] == 5, "average",
                                                      "above")),
               overall_qual = lambda x:
                   np.where(x["overall_qual"] < 5, "below",
                   np.where(x["overall_qual"] == 5, "average",
                                                      "above")),
               kitch_qual = lambda x:
                   np.where((x["kitchen_qual"] == "Po") |
                            (x["kitchen_qual"] == "Fa"), "below",
                   np.where(x["kitchen_qual"] == "TA", "average",
                                                         "above")),
               fplace = lambda x: np.where(x["fireplaces"] > 0, "yes", "no"))
       .drop(columns = cols1_drop))

# Take the following steps:
  # 1. Select columns with certain data types
  # 2. Change the data type of a specific column
  # 3. Replace missing values with 0

data1_num = (data1_all
             .select_dtypes(include = np.number)
             .astype({"id": str})
             .fillna(0))

# Take the following steps:
  # 1. Calculate the Pearson correlation coefficients
  # 2. Sort by the selected column
  # 3. Filter appropriately
  # 4. Select the necessary column
  # 5. Get the index of the column as a list

corr1 = (data1_num
         .corr(numeric_only = True)
         .sort_values(by = "price", ascending = False)
         .query("price > 0.45 | price < -0.45")
         .price
         .index
         .tolist())

# Define the name of the extra column

cols1_extra = "id"

# Append the column

if cols1_extra not in corr1:
    corr1.append(cols1_extra)

# Select the columns that are in the list

data1_num = data1_num[corr1]

# Take the following steps:
  # 1. Create a list
  # 2. Change the data type of a specific column
  # 3. Select columns with certain data types
  # 4. Select the necessary columns

cols1_keep = ["id", "overall_cond", "overall_qual", "kitch_qual", "fplace"]

data1_str = (data1_all
             .astype({"id": str})
             .select_dtypes(include = object)
             [cols1_keep])

# Take the following steps:
  # 1. Create a list
  # 2. Merge the data frames
  # 3. Rename certain columns
  # 4. Drop a specific column
  # 5. Reorder the columns

cols1_order = ["price", "abv_grd_sf", "garage_cars", "garage_sf",
               "bsmt_sf","x1st_flr_sf", "full_bath", "mas_area",
               "totrms_abvgrd", "age", "kitch_qual", "fplace",
               "overall_cond", "overall_qual"]

data2 = (data1_num
         .merge(data1_str, on = "id", how = "left")
         .rename(columns = {"area": "abv_grd_sf",
                            "garage_area": "garage_sf",
                            "total_bsmt_sf": "bsmt_sf",
                            "mas_vnr_area": "mas_area"})
         .drop(columns = "id")
         [cols1_order])

##### Part B - Exploratory Data Analysis #####

# Select columns with certain data types

data2_num = data2.select_dtypes(include = np.number)

# Take the following steps:
  # 1. Create a dictionary
  # 2. Generate the descriptive statistics
  # 3. Rename index

index1_rename = {"count": "Count",
                 "mean": "Mean",
                 "std": "Standard Deviation",
                 "min": "Minimum",
                 "max": "Maximum"}

ds1 = (data2_num
       .agg(["count", "mean", "std", "min", "max"])
       .rename(index = index1_rename))

# Create a new row for the median

ds1.loc["Median"] = data2_num.median()

# Create a new row for the mode

ds1.loc["Mode"] = data2_num.mode().iloc[0]

# Take the following steps:
  # 1. Create a dictionary
  # 2. Rename columns
  # 3. Transpose the data frame
  # 4. Modify the position of columns

cols1_rename = {"price": "Price", "abv_grd_sf": "Abv. Grd. SF",
                "garage_cars": "Garage Doors", "garage_sf": "Garage SF",
                "bsmt_sf": "Basement SF",  "x1st_flr_sf": "1st Flr. SF",
                "full_bath": "Bathrooms", "mas_area": "Masonry SF",
                "totrms_abvgrd": "Rooms Abv. Grd.", "age": "Age"}

ds1 = (ds1
       .rename(columns = cols1_rename)
       .T
       .pipe(mc.MoveToN, "Median", 3)
       .pipe(mc.MoveToN, "Mode", 4))

# Take the following steps:
  # 1. Select columns with certain data types
  # 2. Rename all the columns
  # 3. Calculate the Pearson correlation coefficients

corr2 = (data2
         .select_dtypes(include = np.number)
         .rename(columns = cols1_rename)
         .corr())

# Create a correlation matrix displaying the Pearson correlation coefficients

plt.figure(figsize = (8, 6))
color = sns.diverging_palette(10, 240, as_cmap = True)
cmat = sns.heatmap(corr2, vmin = -1, vmax = 1, annot = True, cmap = color,
                   linewidths = 0.5, linecolor = "black", xticklabels = True,
                   yticklabels = True, annot_kws = {"color": "black"})
cmat.tick_params(bottom = False, left = False)
cmat.set_title("Figure 1 - Correlation Matrix of Numeric Analysis Variables",
               fontdict = {"fontsize": 18}, pad = 12)

plt.show()

# Take the following steps:
  # 1. Modifying existing column
  # 2. Select the necessary columns
  # 3. Calculate Spearman's rank correlation coefficient

corr3 = (data2
         .assign(kitch_qual =
                 np.where(data2["kitch_qual"] == "below", 1,
                 np.where(data2["kitch_qual"] == "average", 2,
                                                            3)))
         [["price", "kitch_qual"]])

corrtest3 = stats.spearmanr(corr3["kitch_qual"], corr3["price"])

# Take the following steps:
  # 1. Modifying existing column
  # 2. Select the necessary columns
  # 3. Calculate the point biserial correlation coefficient

corr4 = (data2
         .assign(fplace = np.where(data2["fplace"] == "yes", 1, 0))
         [["price", "fplace"]])

corrtest4 = stats.pointbiserialr(corr4["fplace"], corr4["price"])

# Take the following steps:
  # 1. Modifying existing column
  # 2. Select the necessary columns
  # 3. Calculate Spearman's rank correlation coefficient

corr5 = (data2
         .assign(overall_cond =
                 np.where(data2["overall_cond"] == "below", 1,
                 np.where(data2["overall_cond"] == "average", 2,
                                                              3)))
         [["price", "overall_cond"]])

corrtest5 = stats.spearmanr(corr5["overall_cond"], corr5["price"])

# Take the following steps:
  # 1. Modifying existing column
  # 2. Select the necessary columns
  # 3. Calculate Spearman's rank correlation coefficient

corr6 = (data2
         .assign(overall_qual =
                 np.where(data2["overall_qual"] == "below", 1,
                 np.where(data2["overall_qual"] == "average", 2,
                                                              3)))
         [["price", "overall_qual"]])

corrtest6 = stats.spearmanr(corr6["overall_qual"], corr6["price"])

# Create a histogram displaying the distribution

(pn.ggplot(data = data2, mapping = pn.aes(x = "price")) +
 pn.geom_histogram(fill = "#005288", color = "black", bins = 25) +
 pn.scale_x_continuous(labels = lambda x: ["${:,.0f}".format(i) for i in x]) +
 pn.labs(title = "Figure 2 - Distribution of Price",
         x = "Price", y = "Frequency") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a new variable

data2["log_price"] = np.log(data2["price"])

# Create a histogram displaying the distribution

(pn.ggplot(data = data2, mapping = pn.aes(x = "log_price")) +
 pn.geom_histogram(fill = "#c0c2c4", color = "black", bins = 25) +
 pn.labs(title = "Figure 3 - Distribution of the Log of Price",
         x = "Log of Price", y = "Frequency") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a scatter plot displaying the relationship between the variables

(pn.ggplot(data = data2, mapping = pn.aes(x = "abv_grd_sf", y = "price")) +
 pn.geom_point(color = "#005288", size = 0.6) +
 pn.geom_smooth(method = "lm", se = False, size = 0.7) +
 pn.scale_x_continuous(labels = lambda x: ["{:,.0f}".format(i) for i in x]) +
 pn.scale_y_continuous(labels = lambda x: ["${:,.0f}".format(i) for i in x]) +
 pn.labs(title = "Figure 4 - Price vs. Above Ground Sq. Ft.",
         x = "Above Ground Sq. Ft.", y = "Price") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a function that identifies outliers using the IQR method

def get_outliers(df, col):
    q1 = df[col].quantile(0.25)
    q3 = df[col].quantile(0.75)
    iqr = q3 - q1
    lower_bound = q1 - (1.5 * iqr)
    upper_bound = q3 + (1.5 * iqr)
    outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]
    return outliers

# Use function to get outliers using the IQR method for two different variables

outliers1 = get_outliers(data2, "abv_grd_sf")

outliers2 = get_outliers(data2, "price")

# Filter appropriately to produce a new data frame

data3 = data2.query("abv_grd_sf < 4000 & price > 15000")

# Create a scatter plot displaying the relationship between the variables

(pn.ggplot(data = data3, mapping = pn.aes(x = "abv_grd_sf", y = "log_price")) +
 pn.geom_point(color = "#0078ae", size = 0.6) +
 pn.geom_smooth(method = "lm", se = False, size = 0.7) +
 pn.scale_x_continuous(labels = lambda x: ["{:,.0f}".format(i) for i in x]) +
 pn.labs(title = "Figure 5 - Log of Price vs. Above Ground Sq. Ft.",
         x = "Above Ground Sq. Ft.", y = " Log of Price") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a scatter plot displaying the relationship between the variables

(pn.ggplot(data = data2, mapping = pn.aes(x = "age", y = "price")) +
 pn.geom_point(color = "#005288", size = 0.6) +
 pn.geom_smooth(method = "lm", se = False, color = "black", size = 0.7) +
 pn.scale_y_continuous(labels = lambda x: ["${:,.0f}".format(i) for i in x]) +
 pn.labs(title = "Figure 6 - Price vs. Age", x = "Age", y = "Price") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a scatter plot displaying the relationship between the variables

(pn.ggplot(data = data3, mapping = pn.aes(x = "age", y = "log_price")) +
 pn.geom_point(color = "#0078ae", size = 0.6) +
 pn.geom_smooth(method = "lm", formula = "y ~ x + I(x**2)", se = False,
                color = "black", size = 0.7) +
 pn.labs(title = "Figure 7 - Log of Price vs. Age",
         x = "Age", y = "Log of Price") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

# Create a new variable

data3["age2"] = np.power(data2["age"], 2)

##### Part C - Statistical Modeling #####

# Take the following steps:
  # 1. Convert categorical variables to dummy variables
  # 2. Drop columns
  # 3. Modify the position of the columns

data3 = (data3
         .join(pd.get_dummies(data3[["kitch_qual", "fplace",
                                     "overall_cond", "overall_qual"]],
                              prefix = ["kitch_qual", "fplace",
                                        "overall_cond", "overall_qual"]))
         .drop(columns = ["kitch_qual", "fplace",
                          "overall_cond", "overall_qual"])
         .pipe(mc.MoveToN, "log_price", 2)
         .pipe(mc.MoveToN, "age2", 12))

### Polynomial Regression

# Take the following steps:
  # 1. Create a list
  # 2. Create a data frame with the necessary independent variables
  # 3. Create a series with just the dependent variable
  # 4. Use hold-out validation to split data into training and testing sets
  # 5. Save function as an object and fit the model to the training data
  # 6. Generate regression output

cols2_keep = ["abv_grd_sf", "garage_sf", "bsmt_sf", "age", "age2",
              "kitch_qual_below", "kitch_qual_average", "fplace_yes",
              "overall_cond_below", "overall_cond_average",
              "overall_qual_below", "overall_qual_average"]

iv1 = data3[cols2_keep]

dv1 = data3["log_price"]

iv1_train, iv1_test, dv1_train, dv1_test = train_test_split(
    iv1, dv1, test_size = 0.3, shuffle = True, random_state = 100)

polreg1 = LinearRegression()

polreg1.fit(iv1_train, dv1_train)

polreg1_output = sm.OLS(dv1_train, sm.add_constant(iv1_train)).fit().summary()

# Take the following steps:
  # 1. Make predictions on the training data
  # 2. Generate the model's residuals
  # 3. Create a data frame with the fitted values and residuals
  # 4. Produce residuals vs. fitted plot
  # 5. Produce a histogram of the residuals
  # 6. Use custom function to get VIFs

polreg1_train_pred = polreg1.predict(iv1_train)

polreg1_train_resid = dv1_train - polreg1_train_pred

rfplot1_df = pd.DataFrame({"fitted": polreg1_train_pred,
                           "resid": polreg1_train_resid})

(pn.ggplot(data = rfplot1_df, mapping = pn.aes(x = "fitted", y = "resid")) +
 pn.geom_point(color = "#005288", size = 0.6) +
 pn.geom_hline(yintercept = 0, color = "grey", linetype = "dashed") +
 pn.labs(title = "Residuals vs. Fitted Plot",
         x = "Fitted Values", y = "Residuals") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

(pn.ggplot(data = rfplot1_df, mapping = pn.aes(x = "resid")) +
 pn.geom_histogram(fill = "#c0c2c4", color = "black", bins = 25) +
 pn.labs(title = "Distribution of Residuals",
         x = "Residuals", y = "Frequency") +
 pn.theme_bw() +
 pn.theme(plot_title = pn.element_text(hjust = 0.5),
          text = pn.element_text(size = 14)))

def compute_vifs(data):
    iv_with_intercept = sm.add_constant(data)
    vifs = [variance_inflation_factor(iv_with_intercept.values, i)
            for i in range(iv_with_intercept.shape[1])]
    vif_series = pd.Series(vifs[1:], index = data.columns)
    return vif_series

compute_vifs(iv1_train.drop(columns = "age2"))

# Take the following steps:
  # 1. Get r-squared
  # 2. Make predictions on the test data
  # 3. Get the RMSE

polreg1.score(iv1_test, dv1_test)

polreg1_test_pred = polreg1.predict(iv1_test)

rmse1 = np.sqrt(mean_squared_error(dv1_test, polreg1_test_pred))

### Lasso Regression

# Take the following steps:
  # 1. Create a data frame with the necessary independent variables
  # 2. Create a series with just the dependent variable
  # 3. Use hold-out validation to split data into training and testing sets
  # 4. Standardize the necessary variables
  # 5. Use 5-fold cross-validation to find optimal hyperparameter
  # 6. Save function as an object and fit the model to the training data
  # 7. Get the r-squared

cols3_keep = ["abv_grd_sf", "garage_cars", "garage_sf", "bsmt_sf",
              "x1st_flr_sf", "full_bath", "mas_area", "totrms_abvgrd",
              "age", "age2", "kitch_qual_below", "kitch_qual_average",
              "fplace_yes", "overall_cond_below", "overall_cond_average",
              "overall_qual_below", "overall_qual_average"]

iv2 = data3[cols3_keep]

dv2 = data3["log_price"]

iv2_train, iv2_test, dv2_train, dv2_test = train_test_split(
    iv2, dv2, test_size = 0.3, shuffle = True, random_state = 100)

def standardize_df(df):
    df_stand = df.copy()
    num_cols = (df_stand.select_dtypes(include = np.number)
                        .select_dtypes(exclude = "uint8").columns)
    scaler = StandardScaler()
    scaler.fit(df_stand[num_cols])
    df_stand[num_cols] = scaler.transform(df_stand[num_cols])
    return df_stand

iv2_train = standardize_df(iv2_train)

iv2_test = standardize_df(iv2_test)

lasso1_cv = LassoCV(cv = 5, random_state = 100)

lasso1_cv.fit(iv2_train, dv2_train)

lasso1 = Lasso(alpha = lasso1_cv.alpha_)

lasso1.fit(iv2_train, dv2_train)

lasso1.score(iv2_train, dv2_train)

# Take the following steps:
  # 1. Get r-squared
  # 2. Make predictions on the test data
  # 3. Get the RMSE

lasso1.score(iv2_test, dv2_test)

lasso1_pred = lasso1.predict(iv2_test)

rmse2 = np.sqrt(mean_squared_error(dv2_test, lasso1_pred))

### K-Nearest Neighbors

# Take the following steps:
  # 1. Use hold-out validation to split data into training and testing sets
  # 2. Save function as an object and perform hyperparameter tuning
  # 3. Fit the model to the training data
  # 4. Get the r-squared

iv3_train, iv3_test, dv3_train, dv3_test = train_test_split(
    iv2, dv2, test_size = 0.3, shuffle = True, random_state = 100)

knn1 = KNeighborsRegressor()

hp1_grid = {"n_neighbors": [4, 8, 12, 16, 20]}

knn1_cv = GridSearchCV(knn1, param_grid = hp1_grid, cv = 5)

knn1_cv.fit(iv3_train, dv3_train)

knn1 = knn1_cv.best_estimator_

knn1.fit(iv3_train, dv3_train)

knn1.score(iv3_train, dv3_train)

# Take the following steps:
  # 1. Get r-squared
  # 2. Make predictions on the test data
  # 3. Get the RMSE

knn1.score(iv3_test, dv3_test)

knn1_test_pred = knn1.predict(iv3_test)

rmse3 = np.sqrt(mean_squared_error(dv3_test, knn1_test_pred))

### Random Forest

# Take the following steps:
  # 1. Use hold-out validation to split data into training and testing sets
  # 2. Perform hyperparameter tuning
  # 3. Save function as an object and fit the model to the training data
  # 4. Get the r-squared

iv4_train, iv4_test, dv4_train, dv4_test = train_test_split(
    iv2, dv2, test_size = 0.3, shuffle = True, random_state = 100)

hp2_grid = {"n_estimators": [200, 250, 500],
            "max_depth": [20, 21, 22],
            "min_samples_split": [4]}

rf1_cv = GridSearchCV(RandomForestRegressor(random_state = 100),
                      hp2_grid, cv = 5)

rf1_cv.fit(iv4_train, dv4_train)

rf1_bp = rf1_cv.best_params_

rf1 = RandomForestRegressor(n_estimators = rf1_bp["n_estimators"],
                            max_depth = rf1_bp["max_depth"],
                            min_samples_split = rf1_bp["min_samples_split"],
                            random_state = 100)

rf1.fit(iv4_train, dv4_train)

rf1.score(iv4_train, dv4_train)

# Take the following steps:
  # 1. Get r-squared
  # 2. Make predictions on the test data
  # 3. Get the RMSE

rf1.score(iv4_test, dv4_test)

rf1_test_pred = rf1.predict(iv4_test)

rmse4 = np.sqrt(mean_squared_error(dv4_test, rf1_test_pred))