from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^coin-offer', views.CoinOfferView.as_view(), name='contact'),
    url(r'^contact', views.ContactUsView.as_view(), name='contact'),
    url(r'^tags', views.TagCoinView.as_view(), name='tags'),
    url(r'^coins', views.CoinsListView.as_view(), name='coins'),
    url(r'^user/create', views.CreateUserView.as_view(), name='create_user'),
    url(r'^user/token', views.CreateTokenView.as_view(), name='create_token'),
]


import json
import argparse
import time
import socket
import select
import logging
import queue

from py_dream_cheeky.button import DreamCheekyButtonThread, ButtonEventType

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()

def run_effects(sock, effects):
    for e in effects:
        if 'suspend' in e:
            send_command(sock, 'SUSPEND', *e['lights'], state=e['suspend'])
            # print("Suspending", e['lights'], e['suspend'])
        if 'props' in e:
            if e.get('duration'):
                send_command(sock, 'EFFECT', *e['lights'], **{
                    prop: {'start': 'current', 'end': value, 'duration': e['duration'], 'overwrite': True}
                    for prop, value in e['props'].items()
                })
                # print("Set effect for props current ->", e['props'], e['duration'], "on lights", e['lights'])
            else:
                send_command(sock, 'SET', *e['lights'], **e['props'])
                # print("Set state", e['props'], "on lights", e['lights'])
        if e.get('wait') and e.get('duration'):
            time.sleep(e['duration'])

def load_config():
    with open('config.json', 'r') as fp:
        return json.load(fp)

def connect(config):
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((config['HOST'], config['PORT']))
        sock.setblocking(0)
        return sock
    except:
        logger.error("Failed to connect to %s:%d", config['HOST'], config['PORT'], exc_info=True)
        return None

def sock_read(sock):
    r, _, _ = select.select([sock], [], [], 0.01)
    data = b''
    if r:
        try:
            data = sock.recv(8192)
        except ConnectionResetError:
            return False
        if len(data) == 0:
            # No data, client is dead
            sock.close()
            return False
    return data

def send_command(sock, command, *args, **kwargs):
    data = {
        'command': command,
        'args': args,
        'kwargs': kwargs
    }
    out = json.dumps(data).encode('utf-8') + b'\n'
    sent = 0
    try:
        while sent < len(out):
            sent += sock.send(out[sent:])
    except socket.error:
        logger.error("Got error from socket %s", sock, exc_info=True)
        sock.close()
        return False
    return True

def run(config, button):
    event_queue = button.get_event_queue()
    click_state = 0
    hold_state = 0
    sock = None

    while True:
        if sock is None:
            sock = connect(config)
            if sock is None:
                time.sleep(1)
                try:
                    while True:
                        event_queue.get(block=False)
                except queue.Empty:
                    pass
                continue

        res = sock_read(sock)
        if res is False or b'QUIT' in res:
            sock = None
            continue

        try:
            event = event_queue.get(timeout=0.01)
        except queue.Empty:
            continue

        if event.type == ButtonEventType.LID_OPEN:
            run_effects(sock, config['EFFECTS'].get('LID_OPEN', []))
        elif event.type == ButtonEventType.BUTTON_CLICK:
            hold_state = 0
            if config['EFFECTS'].get('BUTTON_CLICK'):
                run_effects(sock, config['EFFECTS']['BUTTON_CLICK'][click_state])
                click_state = (click_state + 1) % len(config['EFFECTS']['BUTTON_CLICK'])
        elif event.type == ButtonEventType.BUTTON_HOLD:
            click_state = 0
            if config['EFFECTS'].get('BUTTON_HOLD'):
                run_effects(sock, config['EFFECTS']['BUTTON_HOLD'][hold_state])
                hold_state = (hold_state + 1) % len(config['EFFECTS']['BUTTON_HOLD'])
        elif event.type == ButtonEventType.LID_CLOSE:
            run_effects(sock, config['EFFECTS'].get('LID_CLOSE', []))

def main():
    config = load_config()
    button = DreamCheekyButtonThread(enqueue_events=True)
    button.start()
    try:
        try:
            run(config, button)
        finally:
            button.stop()
    except KeyboardInterrupt:
        # suppress this
        pass

main()


import threading
import socket
import time
import cPickle
import logging
import argparse

class Server():

	def __init__(self, ip, port):
		self.port = int(port)
		self.ip = ip
		self.server_running = True
		self.logger = logging.getLogger("Server")
		logging.basicConfig(level=logging.INFO)

	def start(self):
		self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		address = (self.ip, self.port)
		self.server.bind(address)
		self.server.listen(10)
		self.logger.info(
			"Started listing on {} - {}".format(self.ip, self.port)
		)
		thread = threading.Thread(target = self.server_thread)
		thread.start()

	def server_thread(self):
		while self.server_running:
			try:
				client_connection, client_address = self.server.accept()
				self.logger.info(
					"Got a connection from {} - {}"
					.format(client_address[0], client_address[1])
				)
				thread = threading.Thread(
					target = self.addition_thread,
					args = (client_connection, client_address)
				)
				thread.start()
			except EOFError:
				self.logger.warn("Nothing Received")

	def addition_thread(self, client_connection, client_address):
		data_received = client_connection.recv(1024)
		data = cPickle.loads(data_received)
		self.logger.info("Received {} from client".format(data))
		time.sleep(2)
		sum_data = int(data[0]) + int(data[1])
		client_connection.send(str(sum_data))
		self.logger.info("Result sent to {}".format(client_address[0]))

	def stop(self):
		self.server_running = False
		self.server.shutdown(socket.SHUT_RDWR)
		self.server.close()

if __name__ == '__main__':
	server_detail = argparse.ArgumentParser()
	server_detail.add_argument(
		"-i",
		"--ip",
		help="IP in which server wil run",
		default='127.0.0.1'
	)
	server_detail.add_argument(
		"-p",
		"--port",
		help="Server's Port listening Connection",
		required=True
	)
	server_detail.add_argument(
		"-t",
		"--time",
		help="Time for server will be active in seconds",
		default='150'
	)
	server_detail = server_detail.parse_args()
	server = Server(server_detail.ip, server_detail.port)
	server.start()
	time.sleep(float(server_detail.time))
	server.stop()


"""
649. Binary Tree Upside Down
https://www.lintcode.com/problem/binary-tree-upside-down/description
"""
"""
Definition of TreeNode:
class TreeNode:
    def __init__(self, val):
        self.val = val
        self.left, self.right = None, None
"""

class Solution:
    """
    @param root: the root of binary tree
    @return: new root
    """
    def upsideDownBinaryTree(self, root):
        # write your code here
        root, _ = self.flip(root)
        return root

    def flip(self, root):
        if not root: #处理原始数据传进来就是空的情况
            return root, root
        if not root.left and not root.right:
            return root, root

        new_root, right_leaf = self.flip(root.left)
        right_leaf.left = root.right
        right_leaf.right = root
        root.right = None
        root.left = None
        return new_root, root


import re, sys
from collections import defaultdict
from math import log, ceil
from queue import Queue

EPS = 'ε'
UNDERFLOW = 'Δ'
START = 'Start'
END = 'End'
HUB = 'B'
PUSH = 'PUSH'
POP = 'POP'
NONTERMINAL = re.compile(r'[A-Z][_A-Za-z0-9]*')
TERMINAL = re.compile(r'[Δε_a-z0-9]*')

# Raises an error at a specific line
def error(message: str, line: int):
	raise Exception(f'{message} (line {line})')

class StateCounter:
	"""
	A class for easily generating a next state
	"""
	def __init__(self, n: int = 0):
		self.idx = n - 1
		self.invalid = [self.stateIndex(s.lower()) for s in [START, END]]

	def get(self):
		self.idx += 1
		while self.idx in self.invalid:
			self.idx += 1
		return self.stateName(self.idx)

	def invalidate(self, s: str):
		self.invalid.append(self.stateIndex(s.lower()))

	def stateName(self, n: int):
		if n == 0:
			return 'A'
		res = []
		while n:
			res.append(chr(ord('a') + (n % 26)))
			n //= 26
		return ''.join(res[::-1]).capitalize()

	def stateIndex(self, s: str):
		res, val = 0, 1
		for c in s[::-1]:
			res += val * (ord(c) - ord('a'))
			val *= 26
		return res

class ContextFreeGrammar:
	"""
	A class representing a context free grammar.
	When created, the grammar must be productions
	separated by newlines. Start state must be 'S'.
	States (nonterminals) must be uppercase characters followed by
	a normal variable regex, while a terminals is an all lowercase variable regex.
	Be sure to space-separate terminals and non-terminals.
	"""
	def __init__(self, grammar: str = None):
		self.productions = defaultdict(list)
		self.terminals = set()
		if grammar == None: return

		name = ''
		for idx, line in enumerate(grammar.split('\n')):
			line = line.strip().replace('$', 'ε')
			if not line or line.startswith('#'): continue
			if '->' not in line:
				error(f'Not a valid production: "{line}"', idx + 1)
			if line.startswith('->'):
				line = name + line
			name, rest = line.split('->')
			prods = rest.split('|')
			for prod in prods:
				self.production(name.strip(), prod.split(), idx + 1)

	def production(self, name: str, prod: list, line: int = 0):
		if not NONTERMINAL.fullmatch(name):
			error(f'Invalid production state name: "{name}"', line + 1)
		for term in prod:
			if TERMINAL.fullmatch(term):
				self.terminals.add(term)
			elif not NONTERMINAL.fullmatch(term):
				error(f'Invalid production item: "{term}"', line + 1)

		self.productions[name].append(Production(prod))

	def simplify(self):
		# Merge the start state - it will always be S -> AB
		self.productions[START + END] = self.productions['AB']
		del self.productions['AB']

		# Main simplification - delete epsilons and single terminals
		remove = {'start'}
		replace = {'start'}
		while len(remove) or len(replace):
			remove.clear()
			replace.clear()

			self.removeDuplicates()
			for name, prodlist in self.productions.items():
				if len(prodlist) == 1 and prodlist[0].empty():
					remove.add(name)
				elif len(prodlist) == 1 and prodlist[0].terminal():
					replace.add(name)
			for name, prodlist in self.productions.items():
				for prod in prodlist:
					idx = 0
					while idx < len(prod.terms):
						term = prod.terms[idx]
						if term in remove:
							del prod.terms[idx]
						elif term in replace:
							prod.terms[idx] = self.productions[term][0].terms[0]
							idx += 1
						else:
							idx += 1
					prod.verify()
			for name in remove | replace:
				del self.productions[name]

		# Secondary simplification - Clean up state names
		counter = StateCounter()
		counter.invalidate('S')
		mapping = {}
		mapping[START + END] = 'S'
		for name in self.productions:
			if name not in mapping:
				mapping[name] = counter.get()
		for name, newName in mapping.items():
			self.productions[newName] = self.productions[name]
			del self.productions[name]
			for prod in self.productions[newName]:
				prod.rename(mapping)

	def removeDuplicates(self):
		for name, prodlist in self.productions.items():
			found = set()
			res = []
			for prod in prodlist:
				sprod = str(prod)
				if sprod not in found and sprod != name:
					found.add(sprod)
					res.append(prod)
			self.productions[name] = sorted(res)

	# Name Copyright: Carlos Santana
	def chomskify(self):
		nullable = set()
		for name, prodlist in self.productions.items():
			for prod in prodlist:
				if prod.empty():
					nullable.add(name)

	def toPDA(self):
		return PushdownAutomata(self)

	def __repr__(self):
		nameWidth = max([len(name) for name in self.productions])
		res = []
		for name, prodlist in self.productions.items():
			res.append(f'{name:<{nameWidth}} -> {" | ".join([str(p) for p in prodlist])}')
		return '\n'.join(res)

class Production:
	"""
	A class representing a production in a ContextFreeGrammar.
	Contains the state (nonterminal) name and the rule.
	"""
	def __init__(self, terms: list):
		self.terms = []
		for term in terms:
			if term == UNDERFLOW:
				term = EPS
			if term != EPS:
				self.terms.append(term)
		self.verify()

	def verify(self):
		if len(self.terms) == 0:
			self.terms = [EPS]

	def empty(self):
		return self.terms == [EPS]

	def terminal(self):
		return len(self.terms) == 1 and TERMINAL.fullmatch(self.terms[0])

	def rename(self, mapping: map):
		for i, term in enumerate(self.terms):
			if term in mapping:
				self.terms[i] = mapping[term]

	def __lt__(self, other):
		return str(self) < str(other)

	def __repr__(self):
		return ' '.join(self.terms)

class PushdownAutomata:
	"""
	A class representing a pushdown automata. Generates a graph-like
	structure with transitions as pushes and pops.
	"""
	def __init__(self, grammar: ContextFreeGrammar):
		self.stackSymbols = defaultdict(lambda: defaultdict(list))
		self.names = StateCounter(2)
		self.states = defaultdict(list)
		self.stateNames = set()

		self.transition(START, 'A', EPS, PUSH, UNDERFLOW)
		self.transition('A', HUB, EPS, PUSH, 'S')
		for c in grammar.terminals:
			self.transition(HUB, HUB, c, POP, c)
		self.transition(HUB, END, UNDERFLOW, POP, UNDERFLOW)

		for name, prodlist in grammar.productions.items():
			for prod in prodlist:
				terminal = bool(TERMINAL.fullmatch(prod.terms[0]))

				cur = HUB
				next = HUB if len(prod.terms) == 1 else self.names.get()
				if not terminal:
					self.transition(cur, next, EPS, POP, name)
				else:
					self.transition(cur, next, prod.terms[0], POP, name)
				rest = prod.terms[int(terminal):][::-1]
				for i, term in enumerate(rest):
					if term == EPS: continue
					cur = next
					next = HUB if i == len(rest) - 1 else self.names.get()
					self.transition(cur, next, EPS, PUSH, term)

	def transition(self, cur, next, read, action, value):
		self.stackSymbols[value][action].append([cur, next, read])
		self.states[cur].append(Transition(next, read, action, value))
		self.stateNames.update([cur, next])

	def pathHelper(self, x, y):
		if x == y: return True
		vis = set()
		bfs = Queue()
		bfs.put(x)
		while bfs.qsize():
			cur = bfs.get()
			for trans in self.states[cur]:
				if trans.next in vis: continue
				if trans.next == y:
					return True
				vis.add(trans.next)
				bfs.put(trans.next)
		return False

	def pathExists(self, x, y, z):
		return self.pathHelper(x, y) and self.pathHelper(y, z)

	def toCFG(self):
		cfg = ContextFreeGrammar()
		main = set()

		for sym, value in self.stackSymbols.items():
			for push in value[PUSH]:
				for pop in value[POP]:
					first = f'{push[0]}{pop[1]}'
					second = f'{push[1]}{pop[0]}'
					main.update([first, second])
					cfg.production(first, [push[2], second, pop[2]])

		for x in self.stateNames:
			for y in self.stateNames:
				for z in self.stateNames:
					path, begin, end = x + z, x + y, y + z
					if self.pathExists(x, y, z) and all([x in main for x in [path, begin, end]]):
						if x == z:
							cfg.production(path, [])
						elif x != y and y != z:
							cfg.production(path, [begin, end])

		return cfg

	def __repr__(self):
		nameWidth = max([len(name) for name in self.states])
		return '\n'.join([f'{k:<{nameWidth}} -> {v}' for k, v in self.states.items()])

class Transition:
	"""
	A class representing a transition in a pushdown automata.
	Contains next state, read character, push/pop action, and value to push/pop.
	"""
	def __init__(self, next, read, action, value):
		self.next = next
		self.read = read
		self.action = action
		self.value = value

	def __repr__(self):
		return f'{{{self.next}, {self.read}, {self.action} {self.value}}}'


import time
import random
import asyncio
import aiohttp

URL = "https://api.github.com/events"
MAX_CLIENTS = 3

async def fetch_async(session, pid):
    start = time.time()
    delay = random.randint(2, 5)
    print("Fetch async {} started, delay for {} seconds.".format(pid, delay))
    await asyncio.sleep(delay)
    async with session.get(URL) as response:
        datetime = response.headers.get("Date")
    return "Process #{:<2}: {} took {:.2f} seconds".format(pid, datetime, time.time() - start)

async def asynchronous():
    start = time.time()
    async with aiohttp.ClientSession() as session:
        futures = [fetch_async(session, i) for i in range(1, MAX_CLIENTS + 1)]
        for (i, future) in enumerate(asyncio.as_completed(futures), start=1):
            result = await future
            print("{} {}".format(">>" * i, result))
    print("Process took: {:.2f} seconds".format(time.time() - start))

loop = asyncio.get_event_loop()
print("Process started!")
loop.run_until_complete(asynchronous())
loop.close()


"""
@difficulty: medium
@tags: misc
@notes: Solution 1 use heap to lock the number of items for speeding; Solution 2, sorting all entries at first, count down the use_limit with label.
"""
class Solution:
    def largestValsFromLabels(self, values: List[int], labels: List[int], num_wanted: int, use_limit: int) -> int:
        import heapq
        stats = {}
        for i in range(len(values)):
            v, l = values[i], labels[i]
            if l not in stats:
                stats[l] = [v]
            elif len(stats[l]) < use_limit:
                stats[l].append(v)
            else:
                heapq.heapify(stats[l])
                heapq.heappushpop(stats[l], v)
        res = [0] * num_wanted
        heapq.heapify(res)
        for nums in stats.values():
            for num in nums:
                heapq.heappushpop(res, num)
        return sum(res)


#Author: Jeff Reidy and Noah Wagnon
#Description: Extracts game clock info from cropped
# raw footage

# Importing all necessary libraries
import cv2
import numpy as np
import os
import pytesseract
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import sys

#Enforce that the user provides video path
if (len(sys.argv) < 2):
    print("Please provided video name as command-line-argument")
    exit()

#video path
videoPath = sys.argv[1]

#create the outfile. THis is
outfile = open("ocr_raw_output.txt", 'w')

def get_grayscale(image):
    return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

def thresholding(image):
    return cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]

# Read the video from specified path
cam = cv2.VideoCapture(videoPath)

try:

    # creating a folder named 'RawOCRFrames'
    if not os.path.exists('RawOCRFrames'):
        os.makedirs('RawOCRFrames')

# if not created then raise error
except OSError:
    print ('Error: Creating directory of data')

# frame counters
processedframe = 0
currentframe = 0

while(True):

    # reading from frame
    ret,frame = cam.read()

    if ret:
        # if video is still left continue creating images
        if (currentframe % 30) ==0:
            name = './RawOCRFrames/frame' + str(currentframe) + '.jpg'
#             print ('Creating...' + name)

            # writing the extracted images
            cv2.imwrite(name, frame)

            img1 = get_grayscale(frame)
            img2 = thresholding(img1)

#This is the frame crop the an SEC game
            crop_img = frame[610:655, 865:980]
            name2 = './RawOCRFrames/crop' + str(currentframe) + '.jpg'
#             print ('Creating...' + name2)

            img1 = get_grayscale(crop_img)
            img2 = thresholding(img1)

            img2 = (255- img2)

            #dataPre stores raw result
            dataPre = pytesseract.image_to_string(img2)

            print(dataPre)
            #We will filter and format dataPost
            dataPost =""

            #Loop through and format
            for char in dataPre:
                if (char.isdigit() or char.isspace() ) and not(char == '\n'):
                    dataPost = dataPost + char

            #strip whitespace and ending character
            dataPost = dataPost[:-1]
            dataPost = dataPost.rstrip()
            dataPost = dataPost.lstrip()
            dataPost = dataPost.rstrip('\n')
            outfile.write(str(currentframe) + ' ')

    # Attempting to implement code that writes (0 0 then makes a newline) if OCR detects nothing ie commercials, replays etc.)
            if dataPost and (not dataPost.isspace()) and not (len(dataPost) < 3):

                #detect if play clock is missing and add 00
                if not (' ' in dataPost):
                    dataPost = dataPost + " 0"

                dataPost = dataPost + '\n'
                outfile.write(dataPost)
            else:
                outfile.write("0 0\n")

            # Writes the cropped image to the same folder as the saved frames
            cv2.imwrite(name2, img2)
            processedframe = processedframe + 1

        # increasing counter so that it will
        # show how many frames are created
        currentframe += 1

    else:
        break

outfile.close()
# # Release all space and windows once done

cam.release()
cv2.destroyAllWindows()


"""Константы для jim протокола, настройки"""
# Ключи
ACTION = 'action'
TIME = 'time'
USER = 'user'
ACCOUNT_NAME = 'account_name'
RESPONSE = 'response'
ERROR = 'error'

# Значения
PRESENCE = 'presence'
MESSAGE = 'message'
FROM = 'from'
TO = 'to'

GET_CONTACTS = 'get_contacts'
CONTACT_LIST = 'contact_list'
ADD_CONTACT = 'add_contact'
DEL_CONTACT = 'del_contact'

# Коды ответов (будут дополняться)
BASIC_NOTICE = 100
OK = 200
ACCEPTED = 202
WRONG_REQUEST = 400  # неправильный запрос/json объект
SERVER_ERROR = 500

# Кортеж из кодов ответов
RESPONSE_CODES = (BASIC_NOTICE, OK, ACCEPTED, WRONG_REQUEST, SERVER_ERROR)

# Кортеж действий
ACTIONS = (PRESENCE, MESSAGE, GET_CONTACTS, CONTACT_LIST, ADD_CONTACT, DEL_CONTACT)


'''Arquivo contendo as funções que executam os algoritmos de alinhamento dos textos

A função align_projection, realiza o alinhamento por meio da projeção horizontal da imagem
e depois disso maximiza uma função objetivo, que nesse caso é o erro quadrático.

A função align_hough, realiza o alinhamento por meio da transformada de Hough.

Autor: Felippe Trigueiro Angelo
'''

import numpy as np
from skimage import color, transform, filters, feature, io
import matplotlib.pylab as plt

def border_remove (transformedImage):
    transformedImage2 = np.copy(transformedImage)
    transformedImage3 = np.copy(transformedImage)
    transformedImage2[transformedImage2 == 0] = 1

    transImGray = color.rgb2gray(transformedImage2)
    BintransImage = transImGray <= 0.8
    BintransImage = BintransImage*1

    projHori = np.sum(BintransImage, axis=1) #Soma os valores de uma linha
    projVert = np.sum(BintransImage, axis=0) #Soma os valores de uma coluna

    #Seleciona um retângulo que contenha o texto
    #Nos vetores de projeção, os valores iniciais são baixos devido haverem uma maioria de pixels
    #brancos (branco = 0 nas imagens binárias). Assim, os valores que passam esse limiar são
    #tratados como texto e não como borda resultante da rotação
    projHori[projHori < 40 ] = 0
    projVert[projVert < 14 ] = 0

    sup = np.argmax(projHori > 0) + 1
    left = np.argmax(projVert > 0) + 1

    projHori = projHori[::-1]
    projVert = projVert[::-1]

    inf = np.argmax(projHori > 0) + 1
    right = np.argmax(projVert > 0) + 1

    #Muda a região de fora do retângulo para 1
    transformedImage3[0:(sup-2), :] = 1
    transformedImage3[:, 0:(left-2)] = 1
    transformedImage3[(2-inf):, :] = 1
    transformedImage3[:, (4-right):] = 1

    return transformedImage3

def align_projection(input_Image):
    monImage = color.rgb2gray(input_Image)

    maxHistRotate = np.zeros(2) #Armazena o erro máximo e o respectivo ângulo de rotação
    for k in range(1, 181):
        binImRotate = transform.rotate(monImage, k)
        binImRotate = binImRotate <= 0.8
        binImRotate = binImRotate*1

        projHist = np.sum(binImRotate, axis=1)

        #Cálculo das diferenças entre células adjacentes da projeção
        difProj = projHist[0:(len(projHist)-1)] - projHist[1:len(projHist)]
        squareError = np.sum(difProj*difProj)

        #Verifica qual o erro máximo
        if(squareError > maxHistRotate[0]):
            maxHistRotate[0] = squareError
            maxHistRotate[1] = k

    angle = maxHistRotate[1]
    if (angle > 90 and angle < 180):
       angle = angle + 180

    transformedImage = transform.rotate(input_Image, angle, resize=True)

    #Pós processamento para evitar as bordas resultates da rotação
    transformedImage = border_remove(transformedImage)

    return transformedImage, angle

def align_hough(input_Image):
    #Converte a imagem do padrão RGB para uma imagem binária
    monImage = color.rgb2gray(input_Image)
    monImage = feature.canny(monImage)

    houghTransform, angles, d = transform.hough_line(monImage, theta = np.linspace(-np.pi/2, np.pi/2, 180))
    hTP, angles, d = transform.hough_line_peaks(houghTransform, angles, d)

    angles = np.rad2deg(angles) + 90
    angles = np.round(angles, 0)
    angles = angles.astype(int)

    angle = np.bincount(angles).argmax()

    if (angle > 90 and angle < 180):
        angle = angle + 180

    transformedImage = transform.rotate(input_Image, angle, resize=True)

    #Pós processamento para remoção das bordas resultantes da rotação
    transformedImage = border_remove(transformedImage)

    return transformedImage, angles[0]


import pygame, sys
from pygame.locals import *
import numpy

pygame.init()
clock = pygame.time.Clock()
screen = pygame.display.set_mode((500, 480))

pygame.display.set_caption('Animating Objects')
img = pygame.image.load('head.png')

steps = numpy.linspace(20, 360, 40).astype(int)
right = numpy.zeros((2, len(steps)))
down = numpy.zeros((2, len(steps)))
left = numpy.zeros((2, len(steps)))
up = numpy.zeros((2, len(steps)))

right[0] = steps
right[1] = 20

down[0] = 360
down[1] = steps

left[0] = steps[::-1]
left[1] = 360

up[0] = 20
up[1] = steps[::-1]

pos = numpy.concatenate((right.T, down.T, left.T, up.T))
i = 0

# create a font
font = pygame.font.Font('freesansbold.ttf', 32)
RED = (255, 0, 0)
BLUE = (0, 0, 255)

while True:
    # Erase screen
    screen.fill((0, 0, 0))
    if i >= len(pos):
        i = 0

    screen.blit(img, pos[i])

    # displaying text in the center of the screen
    text = "%d %d %d" % (i, pos[i][0], pos[i][1])
    rendered = font.render(text, True, RED, BLUE)
    screen.blit(rendered, (180, 200))
    i += 1

    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()
            sys.exit()

    pygame.display.update()
    clock.tick(30)


#!/usr/bin/env python3

""" Assignment 3, Exercise 2, INF1340, Fall, 2015. Kanadia

Computer-based immigration office for Kanadia

"""

__author__ = 'Susan Sim'
__email__ = "ses@drsusansim.org"
__copyright__ = "2015 Susan Sim"
__license__ = "MIT License"

import re
import datetime
import json

######################
## global constants ##
######################
REQUIRED_FIELDS = ["passport", "first_name", "last_name",
                   "birth_date", "home", "entry_reason", "from"]

######################
## global variables ##
######################
'''
countries:
dictionary mapping country codes (lowercase strings) to dictionaries
containing the following keys:
"code","name","visitor_visa_required",
"transit_visa_required","medical_advisory"
'''
COUNTRIES = None

#####################
# HELPER FUNCTIONS ##
#####################
def is_more_than_x_years_ago(x, date_string):
    """
    Check if date is less than x years ago.

    :param x: int representing years
    :param date_string: a date string in format "YYYY-mm-dd"
    :return: True if date is less than x years ago; False otherwise.
    """

    now = datetime.datetime.now()
    x_years_ago = now.replace(year=now.year - x)
    date = datetime.datetime.strptime(date_string, '%Y-%m-%d')

    return (date - x_years_ago).total_seconds() < 0

def decide(input_file, countries_file):
    """
    Decides whether a traveller's entry into Kanadia should be accepted

    :param input_file: The name of a JSON formatted file that contains
        cases to decide
    :param countries_file: The name of a JSON formatted file that contains
        country data, such as whether an entry or transit visa is required,
        and whether there is currently a medical advisory
    :return: List of strings. Possible values of strings are:
        "Accept", "Reject", and "Quarantine"
    """

    entries_file = open(input_file)
    entries_string = entries_file.read()
    entries_data = json.loads(entries_string)
    entries_file.close()

    file_of_countries = open(countries_file)
    countries_string = file_of_countries.read()
    global COUNTRIES
    COUNTRIES = json.loads(countries_string)
    file_of_countries.close()
    # create a decision list
    final_decision_list = []
    # check if the entry is in the dictionary
    for entry in entries_data:
        decision_list = []
        # check if the required fields are included
        if not check_required_fields(entry):
            decision_list.append("Reject")
        # check passport format
        if not valid_passport_format(entry['passport']):
            decision_list.append("Reject")
        # check birthday format
        if not valid_date_format(entry['birth_date']):
            decision_list.append("Reject")
        # check if the country is in the list
        if not known_location(entry['from']) or not known_location(entry['home']):
            decision_list.append("Reject")
        # check if there is a via in the entry
        if "via" in entry:
            if not known_location(entry["via"]):
                decision_list.append("Reject")
            if known_location(entry["via"]):
                decision_list.append("Quarantine")
        # check if the contry is Kanada
        if check_home(entry["home"]):
            decision_list.append("Accept")
        # check if it has medical advisory
        if check_quarantine(entry["from"]):
            decision_list.append("Quarantine")
        # check if the person needs visa
        if check_visa_requirement(entry["from"],entry["entry_reason"]):
            if "visa" not in entry:
                decision_list.append("Reject")
            else:
                # reject if the visa is not valid
                if not valid_date_format(entry["visa"]["date"]):
                    decision_list.append("Reject")
                else:
                    # check the visa date
                    if is_more_than_x_years_ago(2,entry["visa"]["date"]):
                        decision_list.append("Reject")
                    # check code format
                    if not valid_visa_format(entry["visa"]["code"]):
                        decision_list.append("Reject")
        # accept if all passed
        decision_list.append("Accept")
        # make the final decision
        final_decision = conflict_decision_resolver(decision_list)
        # add dicesion to the list
        final_decision_list.append(final_decision)

    return final_decision_list

def valid_passport_format(passport_number):
    """
    Checks whether a pasport number is five sets of five alpha-number characters separated by dashes
    :param passport_number: alpha-numeric string
    :return: Boolean; True if the format is valid, False otherwise
    """

    # separate the passport number by dashes
    checklist = passport_number.split('-')

    # if total number of character groups is not five, the format is not valid
    if len(checklist) != 5:
        return False

    # check each group of character
    for i in range(5):

        # if total number of characters in each group is not five, the format is not valid
        if len(checklist[i]) != 5:
            return False

        # if a character is not alphanumeric, the format is not valid
        if not checklist[i].isalnum():
            return False

    # passed all checks, the format is valid
    return True

def valid_visa_format(visa_code):
    """
    Checks whether a visa code is two groups of five alphanumeric characters
    :param visa_code: alphanumeric string
    :return: Boolean; True if the format is valid, False otherwise

    """
    # separate the visa code by dashes
    checklist = visa_code.split('-')

    # if total number of character groups is not five, the format is not valid
    if len(checklist) != 5:
        return False

    # check each group of character
    for i in range(5):

        # if total number of characters in each group is not five, the format is not valid
        if len(checklist[i]) != 5:
            return False

        # if a character is not alphanumeric, the format is not valid
        if not checklist[i].isalnum():
            return False

    # passed all checks, the format is valid
    return True

def valid_date_format(date_string):
    """
    Checks whether a date has the format YYYY-mm-dd in numbers
    :param date_string: date to be checked
    :return: Boolean True if the format is valid, False otherwise
    """
    # separate the date string by dashes
    checklist = date_string.split('-')

    # if total number of character groups is not three, the format is not valid
    if len(checklist) != 3:
        return False

    # if the total number of digits in "year" is not 4 or they are not digits, the format is not valid
    if len(checklist[0]) != 4 or not checklist[0].isdigit():
        return False

    # if the total number of digits in "month" is not 2 or they are not digits, the format is not valid
    if len(checklist[1]) != 2 or not checklist[1].isdigit():
        return False

    # if the total number of digits in "day" is not 2 or they are not digits, the format is not valid
    if len(checklist[2]) != 2 or not checklist[2].isdigit():
        return False

    # passed all checks, the format is valid
    return True

def check_required_fields(single_entry):

    # check if there is one field missing
    for item in REQUIRED_FIELDS:
        if item not in single_entry:
            return False
    return True

def known_location(location):

    # check if the code is in the country list
    code = location["country"].upper()
    if code in COUNTRIES or code == "KAN":
        return True
    return False

def check_quarantine(location):

    # check if it needs medical advisory
    code = location["country"].upper()
    if code in COUNTRIES:
        if COUNTRIES[code]['medical_advisory'] != '':
            return True
    return False

def check_home(location):

    # check if the home is KAN
    code = location["country"].upper()
    return code == "KAN"

def check_visa_requirement(from_location, reason):

    # check if it needs visa
    code = from_location["country"].upper()
    if code in COUNTRIES:
        if COUNTRIES[code]['visitor_visa_required'] == '1' and reason == "visit":
            return True
    return False

def conflict_decision_resolver(list_of_decisions):

    # make decision if has conflict
    if "Quarantine" in list_of_decisions:
        return "Quarantine"
    elif "Reject" in list_of_decisions:
        return "Reject"
    else:
        return "Accept"


#!/home/yli11/.conda/envs/py2/bin/python
import sys
import os
p_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
sys.path.append(os.path.abspath(p_dir+"../utils/"))
from liyc_utils import *
from sklearn import preprocessing
"""Plot high dimension data given data frame

in all input data frame, we assume rows are samples and cols are features.

parameters
----------

By default, no header, no index, all content are numeric.

Users can add their own annotation.

--color_by_a_feature

--size_by_a_feature

--text_by_a_col (will remove this column when doing PCA)

--color_by_a_col  (will remove this column when doing PCA)

for this, please use HEX code, e.g., #12efff

--size_by_a_col  (will remove this column when doing PCA)

UMAP, tSNE parameters

process
-------

log2 transform or not
row_norm or not
col_norm or not

PCA
UMAP
sparsePCA
kernelPCA
PCA using randomized SVD
TruncatedSVD

PCA-100-tSNE
PCA-100-MDS

seaborn.scatterplot or plt.scatter

Each of them have different advantanges

seaborn.scatterplot good for discrete color mapping

https://scikit-learn.org/stable/auto_examples/ensemble/plot_random_forest_embedding.html#sphx-glr-auto-examples-ensemble-plot-random-forest-embedding-py

https://jlmelville.github.io/uwot/abparams.html

https://umap-learn.readthedocs.io/en/latest/parameters.html

"""
current_file_base_name = __file__.split("/")[-1].split(".")[0]

def my_args():
	username = getpass.getuser()
	addon_string = str(uuid.uuid4()).split("-")[-1]
	mainParser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,description="plot heatmap given dataframe.")
	mainParser.add_argument('-j',"--jid",  help="enter a job ID, which is used to make a new directory. Every output will be moved into this folder.", default=current_file_base_name+'_'+username+"_"+str(datetime.date.today()))

	mainParser.add_argument('--remove_zero',  help="remove all rows or cols that are zero", action='store_true')
	mainParser.add_argument('--index',  help=" index is false", action='store_true')
	mainParser.add_argument('--binarize',  help=" force to binary data", action='store_true')
	mainParser.add_argument('--max',  help=" above this value to determine 1", default=1,type=float)
	mainParser.add_argument('--min',  help=" above this value to determine 0", default=0,type=float)
	mainParser.add_argument('--transpose',  help=" df transpose", action='store_true')
	mainParser.add_argument('--sample_norm',  help=" sample norm by sum", action='store_true')
	mainParser.add_argument('-f',"--input",  help="data table input",required=True)
	mainParser.add_argument("--use_cols",  help="use a subset cols",default=None)
	mainParser.add_argument("--index_using",  help="Sometimes we want to show a different label for the indices, then use this option. For example, gene id is unique, however, gene name can be different, it also has upper or lower case problem, in such case, we want to use gene id for data processing and use gene name for visualization.",default="")
	mainParser.add_argument('-s',"--sep",  help="separator",default="\t")

	mainParser.add_argument("--xlabel",default="")
	mainParser.add_argument("--ylabel",default="")
	mainParser.add_argument("--remove_cols",default="")
	mainParser.add_argument("--random_frac",default=None)

	group = mainParser.add_mutually_exclusive_group()
	group.add_argument("--color_using",  help="input a file, index should be the same as the input data frame",default="None")
	group.add_argument("--color_by_a_col",default="None",help="input a column name to be used for coloring")
	group.add_argument("--shape_by_str",default="None",help="input a string to match sample name, matched and unmatched names will have different name; used to show our own data vs public data.")

	mainParser.add_argument("-n",'--maxPC',default=2,type=int)
	mainParser.add_argument("--UMAP_min_dist",default=0.01,type=float)
	mainParser.add_argument("--UMAP_metric",default="euclidean",type=str)
	mainParser.add_argument("--UMAP_n_neighbors",default=15,type=int)
	mainParser.add_argument("--title",default=username+"_"+str(datetime.date.today()))
	mainParser.add_argument("--label_by_last3_element", action='store_true')
	mainParser.add_argument("--smart_label", action='store_true')
	mainParser.add_argument("--kmeans_label", default=-1,type=int)
	mainParser.add_argument("--dbscan_label", action='store_true')
	mainParser.add_argument("--guess_label", action='store_true')
	mainParser.add_argument("--continous", action='store_true')
	# mainParser.add_argument("--gene_filter", action='store_true')
	mainParser.add_argument("--gene_filter_cutoff", type=float,default=-999)
	# https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0149853
	# mainParser.add_argument("--binarize", action='store_true')
	mainParser.add_argument("--label_by_first_element", action='store_true')
	mainParser.add_argument("--label_by_first4_element", action='store_true')
	mainParser.add_argument("--zero_mean_unit_variance", action='store_true')
	mainParser.add_argument("--label_by_meaningful_name", action='store_true')
	mainParser.add_argument("--nPC_list", default=None)
	mainParser.add_argument("--UMAP", action='store_true')
	mainParser.add_argument("--nPCA_UMAP", default=None,type=int)
	mainParser.add_argument("--text", action='store_true')
	mainParser.add_argument("--save_projection_df", action='store_true')
	mainParser.add_argument("--figure_type",default="png",help="pdf,png,jpeg")
	mainParser.add_argument('-o',"--output",  help="output table name",default=current_file_base_name+'_'+username+"_"+str(datetime.date.today()))
	mainParser.add_argument("--header",  help="input table has header", action='store_true')
	mainParser.add_argument("-W", "--width", help="Figure width, by default, w=N_row/4, if given, will replace the default value",type=int,default=500)
	mainParser.add_argument("-H", "--height", help="Figure height, by default, w=N_col/4, if given, will replace the default value",type=int,default=500)
	mainParser.add_argument("--log2_transform",  help="input values will be log2 transformed", action='store_true')

	##------- add parameters above ---------------------
	args = mainParser.parse_args()
	return args

def do_PCA(df,args,n=2,plot=True):
	print ("Performing PCA...")
	pca = PCA(n_components=n)
	print ("input data frame size:%s"%(df.shape[0]))
	# remove_cols = str(args.remove_cols).split(",")
	tmp_df = df.copy()
	print (tmp_df.head())
	# if len(remove_cols)>0:
		# tmp_df = tmp_df.drop([remove_cols],axis=1)
	# tmp_df = df.select_dtypes(['number'])
	print ("numeric data frame size:%s"%(tmp_df.shape[0]))
	if args.color_by_a_col != "None":
		tmp_df = tmp_df.drop([args.color_by_a_col],axis=1)
	flag_nan = tmp_df.isnull().any().any()
	print ("contain NaN values? %s"%(flag_nan))
	if flag_nan:
		print (tmp_df[tmp_df.isna().any(axis=1)].head())
		print ("Will fill NaN by zero")
		tmp_df = tmp_df.fillna(0)
	print ("Input size for PCA is:%s X %s"%(tmp_df.shape[0],tmp_df.shape[1]))
	transform_df_all = pd.DataFrame(pca.fit_transform(tmp_df))
	transform_df = transform_df_all[[0,1]]
	transform_df.columns=['x','y']
	if args.shape_by_str:
		transform_df['symbol'] = [define_shape(x,args.shape_by_str) for x in df.index.tolist()]
	if plot:
		xlabel="PC1 ({0:.1%})".format(pca.explained_variance_ratio_[0])
		ylabel="PC2 ({0:.1%})".format(pca.explained_variance_ratio_[1])
		if args.color_by_a_col == "None":
			transform_df['color']="red"
		else:
			transform_df['color']=df[args.color_by_a_col].tolist()
		transform_df['text']=df.index.tolist()
		transform_df_all['color'] = transform_df['color']
		transform_df_all['text'] = df.index.tolist()
		transform_df['size']=5
		# print (transform_df.head())
		print ("generating figure...")
		if args.save_projection_df:
			transform_df_all.to_csv("%s_projection.csv"%(args.output),index=False)
		plotly_scatter(transform_df,is_discrete=not args.continous,colorscale='Viridis',showlegend=True,xlabel=xlabel,ylabel=ylabel,title="PCA_"+args.title,figure_type=args.figure_type,output=args.output,width=args.width,height=args.height,text=args.text)
	return transform_df

def define_shape(x,q):
	if q in x:
		return "Public"
	else:
		return "Our Own"

def do_UMAP(df,args,n=2,plot=True):
	print ("Performing UMAP...")
	args.output += "_minDist_%s_metric_%s_nNeighbor_%s"%(args.UMAP_min_dist,args.UMAP_metric,args.UMAP_n_neighbors)
	try:
		umap_obj = umap.UMAP(n_components=n,random_state=0,min_dist=args.UMAP_min_dist,metric=args.UMAP_metric,n_neighbors=args.UMAP_n_neighbors)
	except:
		print ("Input UMAP parameters invalid, using default")
		umap_obj = umap.UMAP(n_components=n,random_state=0)
	print ("input data frame size:%s"%(df.shape[0]))
	# remove_cols = str(args.remove_cols).split(",")
	tmp_df = df.copy()
	print (tmp_df.head())
	# if len(remove_cols)>0:
		# tmp_df = tmp_df.drop([remove_cols],axis=1)
	# tmp_df = df.select_dtypes(['number'])
	print ("numeric data frame size:%s"%(tmp_df.shape[0]))
	if args.color_by_a_col != "None":
		tmp_df = tmp_df.drop([args.color_by_a_col],axis=1)
	# print (tmp_df.head())
	flag_nan = tmp_df.isnull().any().any()
	print ("contain NaN values? %s"%(flag_nan))
	if flag_nan:
		print (tmp_df[tmp_df.isna().any(axis=1)].head())
		print ("Will fill NaN by zero")
		tmp_df = tmp_df.fillna(0)
	print ("Input size for UMAP is:%s X %s"%(tmp_df.shape[0],tmp_df.shape[1]))
	transform_df_all = pd.DataFrame(umap_obj.fit_transform(tmp_df.values))
	transform_df = transform_df_all[[0,1]]
	transform_df.columns=['x','y']
	if args.shape_by_str:
		transform_df['symbol'] = [define_shape(x,args.shape_by_str) for x in df.index.tolist()]
	if plot:
		xlabel="UMAP-1"
		ylabel="UMAP-2"
		if args.color_by_a_col == "None":
			transform_df['color']="red"
		else:
			transform_df['color']=df[args.color_by_a_col].tolist()
		transform_df_all['color'] = transform_df['color']
		transform_df_all['text'] = df.index.tolist()
		transform_df['text']=df.index.tolist()
		transform_df['size']=5
		# print (transform_df.head())
		print ("generating figure...")
		if args.save_projection_df:
			transform_df_all.to_csv("%s_projection.csv"%(args.output),index=False)
		args.output = args.output.replace("PCA","UMAP")
		plotly_scatter(transform_df,is_discrete=not args.continous,colorscale='Viridis',showlegend=True,xlabel=xlabel,ylabel=ylabel,title="UMAP_"+args.title,figure_type=args.figure_type,output=args.output,width=args.width,height=args.height,text=args.text)
	return transform_df

def do_nPCA_UMAP(df,args,n=2,plot=True):
	# print ("Performing UMAP...")
	output = args.output+"_nPCA_%s_minDist_%s_metric_%s_nNeighbor_%s"%(args.nPCA_UMAP,args.UMAP_min_dist,args.UMAP_metric,args.UMAP_n_neighbors)
	try:
		umap_obj = umap.UMAP(n_components=n,random_state=0,min_dist=args.UMAP_min_dist,metric=args.UMAP_metric,n_neighbors=args.UMAP_n_neighbors)
	except:
		print ("Input UMAP parameters invalid, using default")
		umap_obj = umap.UMAP(n_components=n,random_state=0)
	print ("input data frame size:%s"%(df.shape[0]))
	# remove_cols = str(args.remove_cols).split(",")

	print ("Performing PCA...")
	pca = PCA(n_components=args.nPCA_UMAP)
	tmp_df = df.copy()
	print (tmp_df.head())
	print ("numeric data frame size:%s"%(tmp_df.shape[0]))
	if args.color_by_a_col != "None":
		tmp_df = tmp_df.drop([args.color_by_a_col],axis=1)
	flag_nan = tmp_df.isnull().any().any()
	print ("contain NaN values? %s"%(flag_nan))
	if flag_nan:
		print (tmp_df[tmp_df.isna().any(axis=1)].head())
		print ("Will fill NaN by zero")
		tmp_df = tmp_df.fillna(0)
	print ("Input size for PCA is:%s X %s"%(tmp_df.shape[0],tmp_df.shape[1]))
	print (tmp_df.head())
	transform_df_all = pd.DataFrame(pca.fit_transform(tmp_df))
	print ("output size for PCA is: %s X %s"%(transform_df_all.shape[0],transform_df_all.shape[1]))

	print ("Performing UMAP...")
	transform_df_all = pd.DataFrame(umap_obj.fit_transform(transform_df_all.values))
	transform_df = transform_df_all[[0,1]]
	transform_df.columns=['x','y']
	print (transform_df.head())
	if args.shape_by_str:
		transform_df['symbol'] = [define_shape(x,args.shape_by_str) for x in df.index.tolist()]
	if plot:
		xlabel="UMAP-1"
		ylabel="UMAP-2"
		if args.color_by_a_col == "None":
			transform_df['color']="red"
		else:
			transform_df['color']=df[args.color_by_a_col].tolist()
		transform_df_all['color'] = transform_df['color']
		transform_df_all['text'] = df.index.tolist()
		transform_df['text']=df.index.tolist()
		transform_df['size']=5
		# print (transform_df.head())
		print ("generating figure...")
		output = output.replace(".html","html")
		print (output)
		if args.save_projection_df:
			transform_df_all.to_csv("%s_projection.csv"%(output),index=False)

		plotly_scatter(transform_df,is_discrete=not args.continous,colorscale='Viridis',showlegend=True,xlabel=xlabel,ylabel=ylabel,title="UMAP_"+args.title,figure_type=args.figure_type,output=output,width=args.width,height=args.height,text=args.text)
	return transform_df

def contains(query,db):
	for s in db:
		if s in query:
			return True
	return False

def smart_label(x):
	x = x.replace(".tpm","").replace(".gz","")
	out = []
	x = x.upper()
	tabu_list = ['SRR','GSM','GEO']
	for i in re.split('_|-|\.|,',x):
		if contains(i,tabu_list):
			continue
		out.append(i)
	return "_".join(out)

def highlight_genes_UMAP(x,y,gene_ids,myDict,nPC=100):
	df = pd.read_csv(x,index_col=0)
	# df = df.transform(lambda x:np.log2(x+1))
	df = df.T
	# print (df.values)
	# tmp_df2 = pd.DataFrame(umap.UMAP(repulsion_strength=5).fit_transform(df.values))
	tmp_df2 = pd.DataFrame(umap.UMAP().fit_transform(df.values))
	# print ("tmp_df2.shape",tmp_df2.shape)
	Parallel(n_jobs=10,verbose=10)(delayed(make_plot)(df,tmp_df2,y,g,myDict[g]) for g in gene_ids)

def highlight_genes_UMAP(x,y,gene_ids,myDict,nPC=100):
	df = pd.read_csv(x,index_col=0)
	# df = df.transform(lambda x:np.log2(x+1))
	df = df.T
	# print (df.values)
	# tmp_df2 = pd.DataFrame(umap.UMAP(repulsion_strength=5).fit_transform(df.values))
	tmp_df2 = pd.DataFrame(umap.UMAP().fit_transform(df.values))
	# print ("tmp_df2.shape",tmp_df2.shape)
	Parallel(n_jobs=10,verbose=10)(delayed(make_plot)(df,tmp_df2,y,g,myDict[g]) for g in gene_ids)

def parse_df(x,y,gene_ids,myDict,nPC=100):
	df = pd.read_csv(x,index_col=0)
	df = df.transform(lambda x:np.log2(x+1))
	df = df.T
	return df

def gene_filter(df,cutoff):
	tmp = df.copy()
	# genes were filtered with TPM value > cutoff in at least 50% of the samples
	# https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0149853
	N = tmp.shape[0]
	tmp[tmp<cutoff]=0
	tmp[tmp>=cutoff]=1
	select_cols = tmp.columns[tmp.sum()>=(N/2.0-0.5)].tolist()
	return df[select_cols]

def get_meaningful_label(x):
	lines = re.split('_|-|\.|,',x)
	elem_list = []
	for l in lines:
		l = l.lower()
		l = l.replace("rnaseq","")
		try:
			t = l.replace("r","")
			t = t.replace("rep","")
			t = t.replace("s","")
			int(t)
		except:
			if "srr" in l:
				continue
			if "geo" in l:
				continue
			if "gse" in l:
				continue
			l.replace("_","")
			# print (l)
			if l == "":
				continue
			elem_list.append(l)
	return "_".join(elem_list)

def binarize_column(myList,min,max):
	out = []
	for v in myList:
		if v >= max:
			out.append(1)
		elif v <=min:
			out.append(0)
		else:
			out.append(-1)
	return out
def remove_zero(df,cutoff=0):
	col_sum = df.sum()
	col_sum2 = col_sum[col_sum>cutoff]
	row_sum = df.sum(axis=1)
	row_sum2 = row_sum[row_sum>cutoff]
	return col_sum2.index.tolist(),row_sum2.index.tolist()
def main():

	args = my_args()
	"""below is the same for very dataframe scripts

	by default our input dataframe is bed format, which is \t separated with no header no index

	"""
	df = general_df_reader(args)
	print (df.head())
	if args.use_cols != None:
		cols = args.use_cols.split(",")
		df = df[cols]
	if args.random_frac:
		df = df.sample(frac = float(args.random_frac))
		args.save_projection_df = True
	# df = df.sample(frac=1)
	if args.binarize:
		df = df.fillna(0)
		tmp = df[args.color_by_a_col].tolist()
		df[args.color_by_a_col] = binarize_column(tmp,args.min,args.max)
		df = df[df[args.color_by_a_col]>=0]
	remove_cols = str(args.remove_cols).split(",")
	try:
		remove_cols.remove("")
	except:
		pass
	if len(remove_cols)>0:
		print (df.head())
		df = df.drop(remove_cols,axis=1)
	if args.transpose:
		df = df.T

	if args.gene_filter_cutoff != -999:
		print ("Filtering genes with above %s in at least half of samples"%(args.gene_filter_cutoff))
		print ("Before dataframe shape:",df.shape)
		df = gene_filter(df,args.gene_filter_cutoff)
		print ("After dataframe shape:",df.shape)
	if args.zero_mean_unit_variance:
		tmp = pd.DataFrame(preprocessing.scale(df))
		tmp.index = df.index.tolist()
		tmp.columns = df.columns.tolist()
		df = tmp.copy()
	if args.label_by_first_element:
		df['group'] = [re.split('_|-|\.|,',x)[0] for x in df.index.tolist()]
		args.color_by_a_col = 'group'
	if args.label_by_first4_element:
		df['group'] = ["_".join(re.split('_|-|\.|,',x)[:4]) for x in df.index.tolist()]
		args.color_by_a_col = 'group'
	if args.kmeans_label >= 2:
		df['group'] = group_similar_names(df.index.tolist(),args.kmeans_label)
		args.color_by_a_col = 'group'
	if args.dbscan_label:
		df['group'] = group_similar_names_dbscan(df.index.tolist())
		args.color_by_a_col = 'group'
	if args.guess_label:
		df['group'] = guess_label(df.index.tolist())
		args.color_by_a_col = 'group'
	if args.smart_label:
		df['group'] = [smart_label(x) for x in df.index.tolist()]
		args.color_by_a_col = 'group'
	if args.label_by_last3_element:
		df['group'] = ["_".join(re.split('_|-|\.|,',x)[-3:]) for x in df.index.tolist()]
		args.color_by_a_col = 'group'
	if args.label_by_meaningful_name:
		df['group'] = [get_meaningful_label(x) for x in df.index.tolist()]
		args.color_by_a_col = 'group'
	if not args.color_using == "None":
		args.input = args.color_using
		color_df = general_df_reader(args)
		print (color_df.head())
		df['group'] = color_df[color_df.columns[-1]]
		args.color_by_a_col = 'group'
	##------- check if jid exist  ----------------------
	# if isdir(args.jid):
		# addon_string = str(uuid.uuid4()).split("-")[-1]
		# print ("The input job id is not available!")
		# args.jid = args.jid+"_"+addon_string
		# print ("The new job id is: "+args.jid)
	# else:
		# print ("The job id is: "+args.jid)
	# os.system("mkdir %s"%(args.jid))

	#-------------- pre-processing ----------------------
	print ("Input data size is:%s X %s"%(df.shape[0],df.shape[1]))
	if args.remove_zero:
		# print (df.head())
		print ("removing zero")
		c,r = remove_zero(df.drop([args.color_by_a_col],axis=1))
		df = df.loc[r,c+[args.color_by_a_col]]
		print ("size after remove zero:",df.shape)

	if args.log2_transform:
		print (df.head())
		if df.shape[1]>100 and args.color_by_a_col!="None":
			df2 = df.drop([args.color_by_a_col],axis=1).transform(lambda x:np.log2(x+1))
			df2[args.color_by_a_col] = df[args.color_by_a_col]
			df = df2
			del df2
			import gc
			gc.collect()
		else:
			for c in df.columns:
				if c == args.color_by_a_col:
					print ("not doing log2 transformation for %s"%(args.color_by_a_col))
					continue
				try:
					df[c] = df[c].astype(float)
					# print (c)
					df[c] = df[c].apply(lambda x:np.log2(x+1))
				except:
					pass

	if args.sample_norm:
		tmp = df.copy()
		print (df.head())
		sel_col = df.columns.tolist()
		sel_col.remove("group")
		tmp = tmp[sel_col]
		tmp = tmp.div(tmp.sum(axis=1), axis=0)
		tmp['group'] = df['group']
		df = tmp.copy()
		# df = df.transform()
	if args.UMAP:
		do_UMAP(df,args,n=args.maxPC)
	elif args.nPCA_UMAP:
		print (args.nPC_list)
		if args.nPC_list:
			for i in args.nPC_list.split(","):
				print ("trying nPC size",i)
				args.nPCA_UMAP = int(i)
				do_nPCA_UMAP(df,args,n=args.maxPC)
		else:
			do_nPCA_UMAP(df,args,n=args.maxPC)

	else:
		do_PCA(df,args,n=args.maxPC)

	## additional commands
	# for f in file_list:
		# os.system("mv %s %s/"%(f,args.jid))
	# os.system("cd %s;mkdir others;mv *.png others;cd others;mv *default* ../"%(args.jid))

if __name__ == "__main__":
	main()


import sys
import argparse
from PttWebCrawler.crawler import PttWebCrawler

__version__ = '1.0-0.1'

def main():
    """The main routine."""
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=(
        "A crawler for the web version of PTT, the largest online community in Taiwan."
        "Input: board name and page indices (or articla ID)"
        "Output: BOARD_NAME-START_INDEX-END_INDEX.json (or BOARD_NAME-ID.json)"
        ))
    parser.add_argument('-b', metavar='BOARD_NAME', help='Board name', required=True)
    parser.add_argument('-o', '--output-dir', help='output directory',
                        default='data')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-i', metavar=('START_INDEX', 'END_INDEX'), type=int, nargs=2, help="Start and end index")
    group.add_argument('-a', metavar='URL', help='article id')
    group.add_argument('-n', metavar='N', help='number of pages to retrieve')
    parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)

    args = parser.parse_args()

    ptt = PttWebCrawler(args.output_dir)
    if args.b:
        board = args.b

    if args.i:
        start = args.i[0]
        if args.i[1] == -1:
            end = ptt.getLastPage(board)
        else:
            end = args.i[1]

        ptt.parse_articles(start, end, board)

    if args.n:
        end = ptt.getLastPage(board)
        start = end - int(args.n) + 1
        ptt.parse_articles(start, end, board)

    if args.a:
        article_id = args.a
        ptt.parse_article(article_id, board)

if __name__ == "__main__":
    main()


"""
MIT License

Copyright (c) 2020-present shay (shayypy)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

------------------------------------------------------------------------------

This project includes code from https://github.com/Rapptz/discord.py, which is
available under the MIT license:

The MIT License (MIT)

Copyright (c) 2015-present Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import abc
from typing import List, Optional

from .activity import Activity
from .asset import Asset
from .colour import Colour
from .embed import _EmptyEmbed, Embed
from .file import MediaType, FileType, File
from .message import HasContentMixin, Message
from .presence import Presence
from .utils import ISO8601

class Messageable(metaclass=abc.ABCMeta):
    """An ABC for models that messages can be sent to.

    The following implement this ABC:

        * :class:`.ChatChannel`
        * :class:`.VoiceChannel`
        * :class:`.Thread`
        * :class:`.DMChannel`
        * :class:`.User`
        * :class:`.Member`
        * :class:`.ext.commands.Context`
    """
    def __init__(self, *, state, data):
        self._state = state
        self.id = data.get('id')
        self._channel_id = data.get('id')

    @property
    def _channel(self):
        if isinstance(self, User):
            return self.dm_channel
        elif hasattr(self, 'channel'):
            return self.channel
        else:
            return self

    async def send(self, *content, **kwargs) -> Message:
        """|coro|

        Send a message to a Guilded channel.

        .. note::
            Guilded supports embeds/attachments/strings in any order, which is
            not practically possible with keyword arguments. For this reason,
            it is recommended that you pass arguments positionally instead.

        .. warning::
            Setting both ``silent`` and ``private`` to ``True`` (a private
            reply with no mention) will not send the reply to the author of
            the message(s) unless they refresh the channel.

        Parameters
        -----------
        content: Union[:class:`str`, :class:`Embed`, :class:`File`, :class:`Emoji`]
            An argument list of the message content, passed in the order that
            each element should display in the message.
        reply_to: List[:class:`Message`]
            A list of up to 5 messages to reply to.
        silent: :class:`bool`
            Whether this reply should not mention the authors of the messages
            it is replying to, if any. Defaults to ``False``. There is an alias
            for this called ``mention_author``, which has the opposite behavior.
        private: :class:`bool`
            Whether this message should only be visible to its author (the
            bot) and the authors of the messages it is replying to. Defaults
            to ``False``. You should not include sensitive data in these
            because private replies can still be visible to server moderators.
        """
        content = list(content)
        if kwargs.get('file'):
            file = kwargs.get('file')
            file.set_media_type(MediaType.attachment)
            if file.url is None:
                await file._upload(self._state)
            content.append(file)
        for file in kwargs.get('files') or []:
            file.set_media_type(MediaType.attachment)
            if file.url is None:
                await file._upload(self._state)
            content.append(file)

        def embed_attachment_uri(embed):
            # pseudo-support attachment:// URI for use in embeds
            for slot in [('image', 'url'), ('thumbnail', 'url'), ('author', 'icon_url'), ('footer', 'icon_url')]:
                url = getattr(getattr(embed, slot[0]), slot[1])
                if isinstance(url, _EmptyEmbed):
                    continue
                if url.startswith('attachment://'):
                    filename = url.strip('attachment://')
                    for node in content:
                        if isinstance(node, File) and node.filename == filename:
                            getattr(embed, f'_{slot[0]}')[slot[1]] = node.url
                            content.remove(node)
                            break

            return embed

        # upload Files passed positionally
        for node in content:
            if isinstance(node, File) and node.url is None:
                node.set_media_type(MediaType.attachment)
                await node._upload(self._state)

        # handle attachment URIs for Embeds passed positionally
        # this is a separate loop to ensure that all files are uploaded first
        for node in content:
            if isinstance(node, Embed):
                content[content.index(node)] = embed_attachment_uri(node)

        if kwargs.get('embed'):
            content.append(embed_attachment_uri(kwargs.get('embed')))

        for embed in kwargs.get('embeds') or []:
            content.append(embed_attachment_uri(embed))

        message_payload = {}
        if kwargs.get('reference') and kwargs.get('reply_to'):
            raise ValueError('Cannot provide both reference and reply_to')

        if kwargs.get('reference'):
            kwargs['reply_to'] = [kwargs['reference'].id]

        if kwargs.get('reply_to'):
            if not isinstance(kwargs['reply_to'], list):
                raise TypeError('reply_to must be type list, not %s' % type(kwargs['reply_to']).__name__)

            message_payload['repliesToIds'] = [message.id for message in kwargs['reply_to']]
            message_payload['isSilent'] = kwargs.get('silent', not kwargs.get('mention_author', True))
            message_payload['isPrivate'] = kwargs.get('private', False)

        share_urls = [message.share_url for message in kwargs.get('share', []) if message.share_url is not None]
        response_coro, payload = self._state.send_message(self._channel_id, content, message_payload, share_urls=share_urls)
        response = await response_coro
        payload['createdAt'] = response.pop('message', response or {}).pop('createdAt', None)
        payload['id'] = payload.pop('messageId')
        try:
            payload['channelId'] = getattr(self, 'id', getattr(self, 'channel', None).id)
        except AttributeError:
            payload['channelId'] = None
        payload['teamId'] = self.team.id if self.team else None
        payload['createdBy'] = self._state.my_id

        author = None
        if payload['teamId'] is not None:
            args = (payload['teamId'], payload['createdBy'])
            try:
                author = self._state._get_team_member(*args) or await self._state.get_team_member(*args, as_object=True)
            except:
                author = None

        if author is None or payload['teamId'] is None:
            try:
                author = self._state._get_user(payload['createdBy']) or await self._state.get_user(payload['createdBy'], as_object=True)
            except:
                author = None

        return self._state.create_message(channel=self._channel, data=payload, author=author)

    async def trigger_typing(self):
        """|coro|

        Begin your typing indicator in this channel.
        """
        return await self._state.trigger_typing(self._channel_id)

    async def history(self, *, limit: int = 50) -> List[Message]:
        """|coro|

        Fetch the message history of this channel.

        Parameters
        -----------
        limit: Optional[:class:`int`]
            The maximum number of messages to fetch. Defaults to 50.

        Returns
        --------
        List[:class:`Message`]
        """
        history = await self._state.get_channel_messages(self._channel_id, limit=limit)
        messages = []
        for message in history.get('messages', []):
            try:
                messages.append(self._state.create_message(channel=self._channel, data=message))
            except:
                pass

        return messages

    async def fetch_message(self, id: str) -> Message:
        """|coro|

        Fetch a message.

        Parameters
        -----------
        id: :class:`str`
            The message's ID to fetch.

        Returns
        --------
        :class:`Message`
            The message from the ID.
        """
        message = await self._state.get_channel_message(self._channel_id, id)
        return message

    async def create_thread(self, *content, **kwargs):
        """|coro|

        Create a new thread in this channel.

        Parameters
        ------------
        content: Any
            The content of the message that should be created as the initial
            message of the newly-created thread. Passing either this or
            ``message`` is required.
        name: :class:`str`
            The name to create the thread with.
        message: Optional[:class:`ChatMessage`]
            The message to create the thread from. Passing either this or
            values for ``content`` is required.
        """
        name = kwargs.get('name')
        message = kwargs.get('message')
        if not name:
            raise TypeError('name is a required argument that is missing.')
        if not message and not content:
            raise TypeError('Must include message, an argument list of content, or both.')

        data = await self._state.create_thread(self._channel_id, content, name=name, initial_message=message)
        thread = self._state.create_channel(data=data.get('thread', data), group=self.group, team=self.team)
        return thread

    async def pins(self):
        """|coro|

        Fetch the pinned messages in this channel.

        Returns
        --------
        List[:class:`Message`]
        """
        messages = []
        data = await self._state.get_pinned_messages(self._channel_id)
        for message_data in data['messages']:
            message = self._state.create_message(data=message_data, channel=self._channel)
            messages.append(message)

        return messages

class User(metaclass=abc.ABCMeta):
    """An ABC for user-type models.

    The following implement this ABC:

        * :class:`guilded.User`
        * :class:`.Member`
        * :class:`.ClientUser`

    Attributes
    -----------
    id: :class:`str`
        The user's id.
    name: :class:`str`
        The user's name.
    subdomain: Optional[:class:`str`]
        The user's "subdomain", or vanity code. Referred to as a "URL" in the
        client.
    email: Optional[:class:`str`]
        The user's email address. This value should only be present when
        accessing this on your own :class:`.ClientUser`\.
    service_email: Optional[:class:`str`]
        The user's "service email".
    bio: :class:`str`
        The user's bio. This is referred to as "About" in the client.
    tagline: :class:`str`
        The user's tagline. This is the text under the user's name on their
        profile page in the client.
    presence: Optional[:class:`Presence`]
        The user's presence.
    dm_channel: Optional[:class:`DMChannel`]
        The user's DM channel with you, if fetched/created and/or cached
        during this session.
    online_at: :class:`datetime.datetime`
        When the user was last online.
    created_at: Optional[:class:`datetime.datetime`]
        When the user's account was created.

        .. warning::
            Due to API ambiguities, this may erroneously be the same as
            :attr:`.joined_at` if this is a :class:`.Member`\.

    blocked_at: Optional[:class:`datetime.datetime`]
        When you blocked the user.
    bot: :class:`bool`
        Whether this user is a bot (Webhook or flow bot).
    moderation_status: Optional[Any]
        The user's moderation status.
    badges: List[:class:`str`]
        The user's badges.
    stonks: Optional[:class:`int`]
        How many "stonks" the user has.
    """
    def __init__(self, *, state, data, **extra):
        self._state = state
        data = data.get('user', data)

        self.type = None
        self.id = data.get('id')
        self.dm_channel = None
        self.name = data.get('name')
        self.colour = Colour(0)
        self.subdomain = data.get('subdomain')
        self.email = data.get('email')
        self.service_email = data.get('serviceEmail')
        self.games = data.get('aliases', [])
        self.bio = (data.get('aboutInfo') or {}).get('bio') or ''
        self.tagline = (data.get('aboutInfo') or {}).get('tagLine') or ''
        self.presence = Presence.from_value(data.get('userPresenceStatus')) or None
        status = data.get('userStatus', {})
        if status.get('content'):
            self.status = Activity.build(status['content'])
        else:
            self.status = None

        self.blocked_at = ISO8601(data.get('blockedDate'))
        self.online_at = ISO8601(data.get('lastOnline'))
        self.created_at = ISO8601(data.get('createdAt') or data.get('joinDate'))
        # in profilev3, createdAt is returned instead of joinDate

        self.avatar_url = Asset('profilePicture', state=self._state, data=data)
        self.banner_url = Asset('profileBanner', state=self._state, data=data)

        self.moderation_status = data.get('moderationStatus')
        self.badges = data.get('badges') or []
        self.stonks = data.get('stonks')

        self.bot = data.get('bot', extra.get('bot', False))

        self.friend_status = extra.get('friend_status')
        self.friend_requested_at = ISO8601(extra.get('friend_created_at'))

    def __str__(self):
        return self.name

    def __eq__(self, other):
        try:
            return self.id == other.id
        except:
            return False

    def __repr__(self):
        return f'<{self.__class__.__name__} id={repr(self.id)} name={repr(self.name)}>'

    @property
    def slug(self):
        return self.subdomain

    @property
    def url(self):
        return self.subdomain

    @property
    def profile_url(self):
        return f'https://guilded.gg/profile/{self.id}'

    @property
    def vanity_url(self):
        if self.subdomain:
            return f'https://guilded.gg/{self.subdomain}'
        else:
            return None

    @property
    def mention(self):
        return f'<@{self.id}>'

    @property
    def display_name(self):
        return self.name

    @property
    def nickname(self):
        return self.name

    @property
    def color(self):
        return self.colour

    @property
    def _channel_id(self):
        return self.dm_channel.id if self.dm_channel else None

    async def create_dm(self) -> Messageable:
        """|coro|

        Create a DM channel with this user.

        Returns
        --------
        :class:`DMChannel`
            The DM channel you created.
        """
        data = await self._state.create_dm_channel([self.id])
        channel = self._state.create_channel(data=data)
        self.dm_channel = channel
        self._state.add_to_dm_channel_cache(channel)
        return channel

    async def hide_dm(self):
        """|coro|

        Visually hide your DM channel with this user in the client.

        Equivalent to :meth:`DMChannel.hide`\.

        Raises
        -------
        ValueError
            Your DM channel with this user is not available or does not exist.
        """
        if self.dm_channel is None:
            raise ValueError('No DM channel is cached for this user. You may want to first run the create_dm coroutine.')

        await self.dm_channel.hide()

    async def send(self, *content, **kwargs) -> Message:
        if self.dm_channel is None:
            await self.create_dm()

        return await super().send(*content, **kwargs)

class TeamChannel(metaclass=abc.ABCMeta):
    """An ABC for the various types of team channels.

    The following implement this ABC:

        * :class:`.ChatChannel`
        * :class:`.DocsChannel`
        * :class:`.ForumChannel`
        * :class:`.VoiceChannel`
        * :class:`.AnnouncementChannel`
        * :class:`.Thread`
    """
    def __init__(self, *, state, group, data, **extra):
        self._state = state
        self.type = None
        data = data.get('data') or data.get('channel') or data
        self.group = group
        self.group_id = data.get('groupId') or getattr(self.group, 'id', None)

        self._team = extra.get('team') or getattr(group, 'team', None)
        self.team_id = data.get('teamId') or self._team.id

        self.id = data.get('id')
        self.name = data.get('name')
        self.position = data.get('priority')
        self.description = data.get('description')
        self.slug = data.get('slug')
        self.roles_synced = data.get('isRoleSynced')
        self.public = data.get('isPublic', False)
        self.settings = data.get('settings')  # no clue

        self.created_at = ISO8601(data.get('createdAt'))
        self.updated_at = ISO8601(data.get('updatedAt'))
        self.added_at = ISO8601(data.get('addedAt'))  # i have no idea what this is
        self.archived_at = ISO8601(data.get('archivedAt'))
        self.auto_archive_at = ISO8601(data.get('autoArchiveAt'))
        created_by = extra.get('created_by') or self._state._get_team_member(self.team_id, extra.get('createdBy'))
        if created_by is None:
            if data.get('createdByInfo'):
                self.created_by = self._state.create_member(data=data.get('createdByInfo'))
        else:
            self.created_by = created_by
        self.archived_by = extra.get('archived_by') or self._state._get_team_member(self.team_id, extra.get('archivedBy'))
        self.created_by_webhook_id = data.get('createdByWebhookId')
        self.archived_by_webhook_id = data.get('archivedByWebhookId')

        self.parent_id = data.get('parentChannelId') or data.get('originatingChannelId')
        # latter is probably only on threads
        if self.parent_id is not None:
            self.parent = self._state._get_team_channel(self.team_id, self.parent_id)
        else:
            self.parent = None

    @property
    def topic(self) -> str:
        return self.description

    @property
    def vanity_url(self) -> str:
        if self.slug and self.team.vanity_url:
            return f'{self.team.vanity_url}/blog/{self.slug}'
        return None

    @property
    def share_url(self) -> str:
        type_ = 'chat'
        if self.type is not None:
            # any type will work for all types of share URLs, but we try
            # to return the 'proper' value here just to be fancy
            type_ = self.type.value

        return f'https://guilded.gg//groups/{self.group_id}/channels/{self.id}/{type_}'

    @property
    def mention(self) -> str:
        return f'<#{self.id}>'

    @property
    def team(self):
        return self._team or self._state._get_team(self.team_id)

    @property
    def guild(self):
        return self.team

    def __str__(self):
        return self.name

    def __repr__(self):
        return f'<{self.__class__.__name__} id={self.id!r} name={self.name!r} team={self.team!r}>'

    def __eq__(self, other):
        try:
            return self.id == other.id
        except:
            return False

    async def delete(self):
        """|coro|

        Delete this channel.
        """
        return await self._state.delete_team_channel(self.team_id, self.group_id, self.id)

class Reply(HasContentMixin, metaclass=abc.ABCMeta):
    """An ABC for replies to posts.

    The following implement this ABC:

        * :class:`.AnnouncementReply`
        * :class:`.DocReply`
        * :class:`.ForumReply`

    .. container:: operations

        .. describe:: x == y

            Checks if two replies are equal.

        .. describe:: x != y

            Checks if two replies are not equal.

    Attributes
    -----------
    id: :class:`int`
        The reply's ID.
    content: :class:`str`
        The reply's content.
    parent: Union[:class:`Announcement`, :class:`Doc`, :class:`ForumTopic`]
        The parent that the reply is under.
    created_at: :class:`datetime.datetime`
        When the reply was created.
    edited_at: Optional[:class:`datetime.datetime`]
        When the reply was last edited.
    deleted_by: Optional[:class:`.Member`]
        Who deleted this reply. This will only be present through a delete
        event, e.g. :func:`on_forum_reply_delete`.
    """
    def __init__(self, *, state, data, parent):
        super().__init__()
        self._state = state
        self.parent = parent
        self.channel = parent.channel
        self.group = parent.group
        self.team = parent.team

        self.id: int = data['id']
        self.content: str = self._get_full_content(data['message'])

        self.author_id: str = data.get('createdBy')
        self.created_by_bot_id: Optional[int] = data.get('createdByBotId')
        self.created_at: datetime.datetime = ISO8601(data.get('createdAt'))
        self.edited_at: Optional[datetime.datetime] = ISO8601(data.get('editedAt'))
        self.deleted_by: Optional[User] = None

        self.replied_to_id: Optional[int] = None
        self.replied_to_author_id: Optional[str] = None

    def __repr__(self):
        return f'<{self.__class__.__name__} id={self.id!r} author={self.author!r} parent={self.parent!r}>'

    def __eq__(self, other):
        return isinstance(other, Reply) and other.id == self.id and other.parent == self.parent

    @property
    def _content_type(self):
        return getattr(self.channel, 'content_type', self.channel.type.value)

    @property
    def author(self) -> Optional[User]:
        """Optional[:class:`.Member`]: The :class:`.Member` that created the
        reply, if they are cached.
        """
        return self.team.get_member(self.author_id)

    @property
    def replied_to(self):
        if self.replied_to_id:
            return self.parent.get_reply(self.replied_to_id)
        return None

    async def add_reaction(self, emoji):
        """|coro|

        Add a reaction to this reply.

        Parameters
        -----------
        emoji: :class:`.Emoji`
            The emoji to add.
        """
        await self._state.add_content_reaction(self._content_type, self.id, emoji.id, reply=True)

    async def remove_self_reaction(self, emoji):
        """|coro|

        Remove your reaction from this reply.

        Parameters
        -----------
        emoji: :class:`.Emoji`
            The emoji to remove.
        """
        await self._state.remove_self_content_reaction(self._content_type, self.id, emoji.id, reply=True)

    async def delete(self):
        """|coro|

        Delete this reply.
        """
        await self._state.delete_content_reply(self._content_type, self.team.id, self.parent.id, self.id)

    async def reply(self, *content, **kwargs):
        """|coro|

        Reply to this reply.

        This method is identical to the reply method of its parent.
        """
        kwargs['reply_to'] = self
        return await self.parent.reply(*content, **kwargs)


import sys

import numpy as np

import theano
import theano.tensor as T
from sklearn.datasets import load_svmlight_file
from sklearn.preprocessing import StandardScaler

sys.path.append('/Users/Andy/Desktop/deep-learning/cost-sensitive/')
from SosRA import SosRA
from toolbox import create_square_cost_mat, class_to_example, make_Z, create_abs_cost_mat

def main():
    # loading data ...
    datasets_dir = './../'
    trX, tr_y = load_svmlight_file(datasets_dir + 'usps_train.txt')
    teX, te_y = load_svmlight_file(datasets_dir + 'usps_test.txt')
    trX = trX.toarray()
    teX = teX.toarray()
    # scaler = StandardScaler()
    # trX = scaler.fit_transform(trX)
    # teX = scaler.transform(teX)

    # experiment setting ...
    n_in = 256
    n_out = 10
    n_hidden = [300]    # 300, 350, 400, 450, 500
    finetune_n_epochs = 1500
    finetune_learning_rate = 0.013
    finetune_batch_size = 10    # 1, 5, 10, ...
    pretrain_n_epochs = 10
    pretrain_learning_rate = 0.01
    pretrain_batch_size = 10
    lamda = 0.
    activation = T.tanh
    numpy_rng = np.random.RandomState(1)
    unsupervised_coef = [0.]
    lamda = [0., 0.]

    # cost_mat = create_square_cost_mat(n_out)
    cost_mat = create_abs_cost_mat(n_out)
    trC = class_to_example(trX, tr_y, cost_mat)
    teC = class_to_example(teX, te_y, cost_mat)
    trZ = make_Z(trC, tr_y)
    teZ = make_Z(teC, te_y)

    reg = SosRA(
        input=T.matrix('input'),
        n_in=n_in,
        n_hidden=n_hidden,
        n_out=n_out,
        activation=activation,
        numpy_rng=numpy_rng,
        unsupervised_coef=unsupervised_coef,
        lamda=lamda
    )

    reg.pre_train(
        train_set=[trX, trC, trZ],
        n_epochs=pretrain_n_epochs,
        learning_rate=pretrain_learning_rate,
        batch_size=pretrain_batch_size
    )

    reg.finetune(
        train_set=[trX, trC, trZ],
        test_set=[teX, teC, teZ],
        n_epochs=finetune_n_epochs,
        learning_rate=finetune_learning_rate,
        batch_size=finetune_batch_size
    )

if __name__ == '__main__':
    main()


from django.db import models
from django.utils import timezone
# Create your models here.
class ArticleCategory(models.Model):
    """
    文章的分类
    """
    #分类的标题
    title = models.CharField(max_length=100,blank=True)
    #分类的创建时间
    created = models.DateTimeField(default=timezone.now)

    #admin站点显示，调试查看对象方便
    def __str__(self):
        return self.title

    class Meta:
        db_table='tb_category'          #修改表名
        verbose_name='类别管理'         #admin站点显示
        verbose_name_plural=verbose_name

from users.models import User
from django.utils import timezone

class Article(models.Model):
    """
    文章
    """
    #定义文章作者， author铜鼓个哦models.Foreignkey外键与内建的User模型联系在一起
    # 参数on_delete用户指定数据删除的方式,避免两个关联表的数据不一致
    author = models.ForeignKey(User,on_delete=models.CASCADE)
    #文章标题图
    avatar = models.ImageField(upload_to='article/%Y%m%d/',blank=True)
    # 文章标题
    title = models.CharField(max_length=20, blank=True)
    #文章栏目的 '一对多'外键      分类
    category = models.ForeignKey(
        ArticleCategory,
        null=True,
        blank=True,
        on_delete=models.CASCADE,
        related_name='article'
    )
    # 文章标签
    tags = models.CharField(max_length=20,blank=True)
    #概要
    sumary=models.CharField(max_length=200,null=False,blank=False)
    #文章正文
    content=models.TextField()
    #浏览量
    total_views=models.PositiveIntegerField(default=0)
    #文章评论数
    comments_count=models.PositiveIntegerField(default=0)
    #文章创建时间
    #参数 default=timezone.now指定其在创建时将默认写入当前的时间
    created = models.DateTimeField(default=timezone.now)
    #文章更新时间
    #参数auto_now=True指定每次数据更新时写入当前时间
    updated=models.DateTimeField(auto_now=True)
    #内部类class Meta用于给model定义元数据
    class Meta:
        # ordering 指定模型返回的数据的排列顺序
        # '-created' 表明数据应该以倒序排列
        db_table='tb_article'
        ordering=('-created',)
        verbose_name='文章管理'
        verbose_name_plural=verbose_name
        # 函数 __str__ 定义当调用对象的 str() 方法时的返回值内容
        # 它最常见的就是在Django管理后台中做为对象的显示值。因此应该总是为 __str__

    def __str__(self):
        #将文章标题返回
        return self.title

class Comment(models.Model):
    """ 评论"""
    #评论内容
    content=models.TextField()
    #评论的文章
    article=models.ForeignKey(Article,
                              on_delete=models.SET_NULL,
                              null=True)
    #发表评论的用户
    user=models.ForeignKey('users.User',
                           on_delete=models.SET_NULL,
                           null=True)
    #评论发布时间
    created=models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.article.title

    class Meta:
        db_table='tb_comment'
        verbose_name = '评论管理'
        verbose_name_plural = verbose_name


#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import pickle
import random
import time
import tempfile
import unittest

from py4j.protocol import Py4JJavaError

from pyspark import SparkConf, SparkContext, Broadcast
from pyspark.java_gateway import launch_gateway
from pyspark.serializers import ChunkedStream
from pyspark.sql import SparkSession, Row

class BroadcastTest(unittest.TestCase):
    def tearDown(self):
        if getattr(self, "sc", None) is not None:
            self.sc.stop()
            self.sc = None

    def _test_encryption_helper(self, vs):
        """
        Creates a broadcast variables for each value in vs, and runs a simple job to make sure the
        value is the same when it's read in the executors.  Also makes sure there are no task
        failures.
        """
        bs = [self.sc.broadcast(value=v) for v in vs]
        exec_values = self.sc.parallelize(range(2)).map(lambda x: [b.value for b in bs]).collect()
        for ev in exec_values:
            self.assertEqual(ev, vs)
        # make sure there are no task failures
        status = self.sc.statusTracker()
        for jid in status.getJobIdsForGroup():
            for sid in status.getJobInfo(jid).stageIds:
                stage_info = status.getStageInfo(sid)
                self.assertEqual(0, stage_info.numFailedTasks)

    def _test_multiple_broadcasts(self, *extra_confs):
        """
        Test broadcast variables make it OK to the executors.  Tests multiple broadcast variables,
        and also multiple jobs.
        """
        conf = SparkConf()
        for key, value in extra_confs:
            conf.set(key, value)
        conf.setMaster("local-cluster[2,1,1024]")
        self.sc = SparkContext(conf=conf)
        self._test_encryption_helper([5])
        self._test_encryption_helper([5, 10, 20])

    def test_broadcast_with_encryption(self):
        self._test_multiple_broadcasts(("spark.io.encryption.enabled", "true"))

    def test_broadcast_no_encryption(self):
        self._test_multiple_broadcasts()

    def _test_broadcast_on_driver(self, *extra_confs):
        conf = SparkConf()
        for key, value in extra_confs:
            conf.set(key, value)
        conf.setMaster("local-cluster[2,1,1024]")
        self.sc = SparkContext(conf=conf)
        bs = self.sc.broadcast(value=5)
        self.assertEqual(5, bs.value)

    def test_broadcast_value_driver_no_encryption(self):
        self._test_broadcast_on_driver()

    def test_broadcast_value_driver_encryption(self):
        self._test_broadcast_on_driver(("spark.io.encryption.enabled", "true"))

    def test_broadcast_value_against_gc(self):
        # Test broadcast value against gc.
        conf = SparkConf()
        conf.setMaster("local[1,1]")
        conf.set("spark.memory.fraction", "0.0001")
        self.sc = SparkContext(conf=conf)
        b = self.sc.broadcast([100])
        try:
            res = self.sc.parallelize([0], 1).map(lambda x: 0 if x == 0 else b.value[0]).collect()
            self.assertEqual([0], res)
            self.sc._jvm.java.lang.System.gc()
            time.sleep(5)
            res = self.sc.parallelize([1], 1).map(lambda x: 0 if x == 0 else b.value[0]).collect()
            self.assertEqual([100], res)
        finally:
            b.destroy()

    def test_broadcast_when_sc_none(self):
        # SPARK-39029 : Test case to improve test coverage of broadcast.py
        # It tests the case when SparkContext is none and Broadcast is called at executor
        conf = SparkConf()
        conf.setMaster("local-cluster[2,1,1024]")
        self.sc = SparkContext(conf=conf)
        bs = self.sc.broadcast([10])
        bs_sc_none = Broadcast(sc=None, path=bs._path)
        self.assertEqual(bs_sc_none.value, [10])

    def test_broadcast_for_error_condition(self):
        # SPARK-39029: Test case to improve test coverage of broadcast.py
        # It tests the case when broadcast should raise error .
        conf = SparkConf()
        conf.setMaster("local-cluster[2,1,1024]")
        self.sc = SparkContext(conf=conf)
        bs = self.sc.broadcast([1])
        with self.assertRaisesRegex(pickle.PickleError, "Could.*not.*serialize.*broadcast"):
            self.sc.broadcast(self.sc)
        with self.assertRaisesRegex(Py4JJavaError, "RuntimeError.*Broadcast.*destroyed.*driver"):
            self.sc.parallelize([1]).map(lambda x: bs.destroy()).collect()
        with self.assertRaisesRegex(Py4JJavaError, "RuntimeError.*Broadcast.*unpersisted.*driver"):
            self.sc.parallelize([1]).map(lambda x: bs.unpersist()).collect()

    def test_broadcast_in_udfs_with_encryption(self):
        conf = SparkConf()
        conf.set("spark.io.encryption.enabled", "true")
        conf.setMaster("local-cluster[2,1,1024]")
        self.sc = SparkContext(conf=conf)
        bar = {"a": "aa", "b": "bb"}
        foo = self.sc.broadcast(bar)
        spark = SparkSession(self.sc)
        spark.udf.register("MYUDF", lambda x: foo.value[x] if x else "")
        sel = spark.sql("SELECT MYUDF('a') AS a, MYUDF('b') AS b")
        self.assertEqual(sel.collect(), [Row(a="aa", b="bb")])
        spark.stop()

class BroadcastFrameProtocolTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        gateway = launch_gateway(SparkConf())
        cls._jvm = gateway.jvm
        cls.longMessage = True
        random.seed(42)

    def _test_chunked_stream(self, data, py_buf_size):
        # write data using the chunked protocol from python.
        chunked_file = tempfile.NamedTemporaryFile(delete=False)
        dechunked_file = tempfile.NamedTemporaryFile(delete=False)
        dechunked_file.close()
        try:
            out = ChunkedStream(chunked_file, py_buf_size)
            out.write(data)
            out.close()
            # now try to read it in java
            jin = self._jvm.java.io.FileInputStream(chunked_file.name)
            jout = self._jvm.java.io.FileOutputStream(dechunked_file.name)
            self._jvm.DechunkedInputStream.dechunkAndCopyToOutput(jin, jout)
            # java should have decoded it back to the original data
            self.assertEqual(len(data), os.stat(dechunked_file.name).st_size)
            with open(dechunked_file.name, "rb") as f:
                byte = f.read(1)
                idx = 0
                while byte:
                    self.assertEqual(data[idx], bytearray(byte)[0], msg="idx = " + str(idx))
                    byte = f.read(1)
                    idx += 1
        finally:
            os.unlink(chunked_file.name)
            os.unlink(dechunked_file.name)

    def test_chunked_stream(self):
        def random_bytes(n):
            return bytearray(random.getrandbits(8) for _ in range(n))

        for data_length in [1, 10, 100, 10000]:
            for buffer_length in [1, 2, 5, 8192]:
                self._test_chunked_stream(random_bytes(data_length), buffer_length)

if __name__ == "__main__":
    from pyspark.tests.test_broadcast import *  # noqa: F401

    try:
        import xmlrunner

        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
    except ImportError:
        testRunner = None
    unittest.main(testRunner=testRunner, verbosity=2)


import sys
import os

from PIL import Image

# use sys module, grab the first and second argument
image_folder = sys.argv[1]
output_folder = sys.argv[2]

print(image_folder, output_folder)
#check if folder exists , if not, create it

if not os.path.exists(output_folder):
  os.makedirs(output_folder)

#loop through the entire folder and then convert to .png
for filename in os.listdir(image_folder):
  img = Image.open(f'{image_folder}{filename}')
  clean_name = os.path.splitext(filename)
  print(clean_name)

  img.save(f'{output_folder}{filename}','png')
  print('all done!')

#save them to the new folder