#!/usr/bin/env python3

################################################################################
## {Description}: Accessing raspicam/usbcam
################################################################################
## Author: Khairul Izwan Bin Kamsani
## Version: {1}.{0}.{0}
## Email: {wansnap@gmail.com}
################################################################################

# import the necessary Python packages
from __future__ import print_function
from __future__ import division
import sys
import cv2
import time
import imutils

# import the necessary ROS packages
from std_msgs.msg import String
from std_msgs.msg import Bool
from std_msgs.msg import Int64
from std_msgs.msg import Float64
from sensor_msgs.msg import Image
from sensor_msgs.msg import CameraInfo

from cv_bridge import CvBridge
from cv_bridge import CvBridgeError

import rospy

class CameraPreview:
	def __init__(self):

		self.bridge = CvBridge()
		self.image_received = False

		self.alpha = 0.5

		self.pump0 = Bool()
		self.pump1 = Bool()
		self.pump2 = Bool()
		self.pump3 = Bool()
		self.fan = Bool()
		self.light = Bool()

		self.light.data = False
		self.lightStatus = False

		self.fan.data = False
		self.fanStatus = False

		rospy.logwarn("CameraPreview Node [ONLINE]...")

		# rospy shutdown
		rospy.on_shutdown(self.cbShutdown)

		# Subscribe to Image msg
		self.image_topic = "/cv_camera_nuc/image_raw"
#		self.image_topic = "/camera/rgb/image_raw"
		self.image_sub = rospy.Subscriber(self.image_topic, Image, self.cbImage)

		# Subscribe to CameraInfo msg
		self.cameraInfo_topic = "/cv_camera_nuc/camera_info"
#		self.cameraInfo_topic = "/camera/rgb/camera_info"
		self.cameraInfo_sub = rospy.Subscriber(self.cameraInfo_topic, CameraInfo,
			self.cbCameraInfo)

		# Subscribe to Int64 msg
		self.fingerTipX_topic = "/fingerTipX"
		self.fingerTipX_sub = rospy.Subscriber(self.fingerTipX_topic, Int64,
			self.cbFingerTipX)

		# Subscribe to Int64 msg
		self.fingerTipY_topic = "/fingerTipY"
		self.fingerTipY_sub = rospy.Subscriber(self.fingerTipY_topic, Int64,
			self.cbFingerTipY)

		# Subscribe to Int64 msg
		self.valSoil0_topic = "/val_soil0"
		self.valSoil0_sub = rospy.Subscriber(self.valSoil0_topic, Int64,
			self.cbSoil0)

		# Subscribe to Int64 msg
		self.valSoil1_topic = "/val_soil1"
		self.valSoil1_sub = rospy.Subscriber(self.valSoil1_topic, Int64,
			self.cbSoil1)

		# Subscribe to Int64 msg
		self.valSoil2_topic = "/val_soil2"
		self.valSoil2_sub = rospy.Subscriber(self.valSoil2_topic, Int64,
			self.cbSoil2)

		# Subscribe to Int64 msg
		self.valSoil3_topic = "/val_soil3"
		self.valSoil3_sub = rospy.Subscriber(self.valSoil3_topic, Int64,
			self.cbSoil3)

		# Subscribe to Float64 msg
		self.valHumid_topic = "/val_Humid"
		self.valHumid_sub = rospy.Subscriber(self.valHumid_topic, Float64,
			self.cbHumid)

		# Subscribe to Float64 msg
		self.valTemp_topic = "/val_Temp"
		self.valTemp_sub = rospy.Subscriber(self.valTemp_topic, Float64,
			self.cbTemp)

		# Subscribe to Bool msg
		self.valPump0_topic = "/pump0"
		self.valPump0_sub = rospy.Subscriber(self.valPump0_topic, Bool,
			self.cbValPump0)

		# Subscribe to Bool msg
		self.valPump1_topic = "/pump1"
		self.valPump1_sub = rospy.Subscriber(self.valPump1_topic, Bool,
			self.cbValPump1)

		# Subscribe to Bool msg
		self.valPump2_topic = "/pump2"
		self.valPump2_sub = rospy.Subscriber(self.valPump2_topic, Bool,
			self.cbValPump2)

		# Subscribe to Bool msg
		self.valPump3_topic = "/pump3"
		self.valPump3_sub = rospy.Subscriber(self.valPump3_topic, Bool,
			self.cbValPump3)

		# Publish to Bool msg
		self.pump0_topic = "/pump0"
		self.pump0_pub = rospy.Publisher(self.pump0_topic, Bool,
			queue_size=10)

		# Publish to Bool msg
		self.pump1_topic = "/pump1"
		self.pump1_pub = rospy.Publisher(self.pump1_topic, Bool,
			queue_size=10)

		# Publish to Bool msg
		self.pump2_topic = "/pump2"
		self.pump2_pub = rospy.Publisher(self.pump2_topic, Bool,
			queue_size=10)

		# Publish to Bool msg
		self.pump3_topic = "/pump3"
		self.pump3_pub = rospy.Publisher(self.pump3_topic, Bool,
			queue_size=10)

		# Publish to Bool msg
		self.light_topic = "/light"
		self.light_pub = rospy.Publisher(self.light_topic, Bool,
			queue_size=10)

		# Publish to Bool msg
		self.fan_topic = "/fan"
		self.fan_pub = rospy.Publisher(self.fan_topic, Bool,
			queue_size=10)

		# Allow up to one second to connection
		rospy.sleep(2)

	def cbHumid(self, msg):
		self.valHumid = msg.data

	def cbTemp(self, msg):
		self.valTemp = msg.data

	def cbValPump0(self, msg):
		self.valPump0 = msg.data

	def cbValPump1(self, msg):
		self.valPump1 = msg.data

	def cbValPump2(self, msg):
		self.valPump2 = msg.data

	def cbValPump3(self, msg):
		self.valPump3 = msg.data

	def cbSoil0(self, msg):
		self.soil0 = msg.data

	def cbSoil1(self, msg):
		self.soil1 = msg.data

	def cbSoil2(self, msg):
		self.soil2 = msg.data

	def cbSoil3(self, msg):
		self.soil3 = msg.data

	def cbFingerTipX(self, msg):
		self.fingerTipX = msg.data

	def cbFingerTipY(self, msg):
		self.fingerTipY = msg.data

	# Convert image to OpenCV format
	def cbImage(self, msg):

		try:
			self.cv_image = self.bridge.imgmsg_to_cv2(msg, "bgr8")

			# comment if the image is mirrored
#			self.cv_image = cv2.flip(self.cv_image, 1)
		except CvBridgeError as e:
			print(e)

		if self.cv_image is not None:
			self.image_received = True
		else:
			self.image_received = False

	# Get CameraInfo
	def cbCameraInfo(self, msg):

		self.imgWidth = msg.width
		self.imgHeight = msg.height

	# Image information callback
	def cbInfo(self):

		self.fontFace = cv2.FONT_HERSHEY_DUPLEX
		self.fontScale = 0.5
		self.color = (255, 255, 255)
		self.color1 = (0, 255, 0)
		self.thickness = 1
		self.lineType = cv2.LINE_AA
		self.bottomLeftOrigin = False # if True (text upside down)

		self.timestr = time.strftime("%Y%m%d-%H:%M:%S")

#		cv2.putText(self.cv_image, "{}".format(self.timestr), (10, 20),
#			self.fontFace, self.fontScale, self.color, self.thickness, self.lineType,
#			self.bottomLeftOrigin)
#		cv2.putText(self.cv_image, "Sample", (10, self.imgHeight-10),
#			self.fontFace, self.fontScale, self.color, self.thickness, self.lineType,
#			self.bottomLeftOrigin)
#		cv2.putText(self.cv_image, "(Temp: %.2f, Humidiity: %.2f)" % (self.imgWidth, self.imgHeight),
#			(self.imgWidth-100, self.imgHeight-10), self.fontFace, self.fontScale,
#			self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
		cv2.putText(self.cv_image, "(Temp: %.2f, Humidiity: %.2f)" % (self.valTemp, self.valHumid),
			(10, 20), self.fontFace, self.fontScale,
			self.color, self.thickness, self.lineType, self.bottomLeftOrigin)

	# Show the output frame
	def cbShowImage(self):
#		self.cv_image_clone = imutils.resize(
#						self.cv_image.copy(),
#						width=320
#						)

#		cv2.imshow("CameraPreview", self.cv_image_clone)
		cv2.imshow("CameraPreview", self.cv_image)
		cv2.waitKey(1)

	# Preview image + info
	def cbPreview(self):

		if self.image_received:
			self.cbInfo()
			self.cbDrawBorder()
			self.cbShowImage()
		else:
			rospy.logerr("No images recieved")

	# rospy shutdown callback
	def cbShutdown(self):

		rospy.logerr("CameraPreview Node [OFFLINE]...")
		cv2.destroyAllWindows()

	# draw border
	def cbDrawBorder(self):
		overlay = self.cv_image.copy()
#		output = self.cv_image.copy()

#		cv2.line(overlay, (0, self.imgHeight // 2), (self.imgWidth, self.imgHeight // 2), (0, 0, 255), 3)

#		cv2.line(overlay, (self.imgWidth // 4, self.imgHeight // 2), (self.imgWidth, self.imgHeight // 2), (0, 0, 255), 3)
		cv2.putText(overlay, "Light",
				(self.imgWidth//2 - 200, self.imgHeight//2 + 200), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
		if (self.fingerTipX <= self.imgWidth//2 - 200 + 20 and self.fingerTipX >= self.imgWidth//2 - 200 - 20) and \
			(self.fingerTipY <= self.imgHeight//2 + 200 + 20 and self.fingerTipY >= self.imgHeight//2 + 200 - 20):
			if self.light.data == False and self.lightStatus == False:
				self.light.data = True
			else:
				self.light.data = False
#			cv2.circle(overlay, (self.imgWidth//2 - 200, self.imgHeight//2 + 200), 30, (255, 0, 255), -1)
#			cv2.putText(overlay, "P6",
#				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
#				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
#			cv2.putText(overlay, "Red Chillis",
#				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
#				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass
#			self.light.data = False

		cv2.putText(overlay, "Fan",
				(self.imgWidth//2 + 200, self.imgHeight//2 + 200), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
		if (self.fingerTipX <= self.imgWidth//2 + 200 + 20 and self.fingerTipX >= self.imgWidth//2 + 200 - 20) and \
			(self.fingerTipY <= self.imgHeight//2 + 200 + 20 and self.fingerTipY >= self.imgHeight//2 + 200 - 20):
			if self.fan.data == False and self.fanStatus == False:
				self.fan.data = True
			else:
				self.fan.data = False
#			cv2.circle(overlay, (self.imgWidth//2 - 200, self.imgHeight//2 + 200), 30, (255, 0, 255), -1)
#			cv2.putText(overlay, "P6",
#				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
#				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
#			cv2.putText(overlay, "Red Chillis",
#				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
#				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass
#		cv2.circle(overlay, (self.imgWidth//2 + 200, self.imgHeight//2 + 200), 30, (255, 0, 255), -1)

		if (self.fingerTipX <= 150 + 20 and self.fingerTipX >= 150 - 20) and \
			(self.fingerTipY <= 150 + 20 and self.fingerTipY >= 150 - 20):
			cv2.circle(overlay, (150, 150), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P6",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

		if (self.fingerTipX <= 150 + 20 and self.fingerTipX >= 150 - 20) and \
			(self.fingerTipY <= 300 + 20 and self.fingerTipY >= 300 - 20):
			cv2.circle(overlay, (150, 300), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P5",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

		if (self.fingerTipX <= 250 + 20 and self.fingerTipX >= 250 - 20) and \
			(self.fingerTipY <= 150 + 20 and self.fingerTipY >= 150 - 20):
			cv2.circle(overlay, (250, 150), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P4",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

		if (self.fingerTipX <= 250 + 20 and self.fingerTipX >= 250 - 20) and \
			(self.fingerTipY <= 300 + 20 and self.fingerTipY >= 300 - 20):
			cv2.circle(overlay, (250, 300), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P4",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

#		cv2.circle(overlay, (400, 150), 20, (0, 0, 255), -1)

		if (self.fingerTipX <= 350 + 20 and self.fingerTipX >= 350 - 20) and \
			(self.fingerTipY <= 300 + 20 and self.fingerTipY >= 300 - 20):
			cv2.circle(overlay, (350, 300), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P3",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

		if (self.fingerTipX <= 500 + 20 and self.fingerTipX >= 500 - 20) and \
			(self.fingerTipY <= 150 + 20 and self.fingerTipY >= 150 - 20):
			cv2.circle(overlay, (500, 150), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P2",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass

		if (self.fingerTipX <= 500 + 20 and self.fingerTipX >= 500 - 20) and \
			(self.fingerTipY <= 300 + 20 and self.fingerTipY >= 300 - 20):
			cv2.circle(overlay, (500, 300), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "P1",
				(self.fingerTipX, self.fingerTipY + 40), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
			cv2.putText(overlay, "Red Chillis",
				(self.fingerTipX, self.fingerTipY + 60), self.fontFace, self.fontScale,
				self.color1, self.thickness, self.lineType, self.bottomLeftOrigin)
		else:
			pass
#
		if (self.fingerTipX <= 600 + 20 and self.fingerTipX >= 600 - 20) and \
			(self.fingerTipY <= 400 + 20 and self.fingerTipY >= 400 - 20):
			cv2.circle(overlay, (600, 400), 20, (0, 255, 0), -1) #3
			cv2.putText(overlay, "%s" % (self.valPump3),
				(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
			self.pump3.data = True
		else:
			cv2.circle(overlay, (600, 400), 20, (0, 0, 255), -1) #3
			self.pump3.data = False

		if (self.fingerTipX <= 600 + 20 and self.fingerTipX >= 600 - 20) and \
			(self.fingerTipY <= 60 + 20 and self.fingerTipY >= 60 - 20):
			cv2.circle(overlay, (600, 60), 20, (0, 255, 0), -1) #2
			cv2.putText(overlay, "%s" % (self.valPump2),
				(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
			self.pump2.data = True
		else:
			cv2.circle(overlay, (600, 60), 20, (0, 0, 255), -1) #2
			self.pump2.data = False

		if (self.fingerTipX <= 20 + 20 and self.fingerTipX >= 20 - 20) and \
			(self.fingerTipY <= 340 + 20 and self.fingerTipY >= 340 - 20):
			cv2.circle(overlay, (20, 340), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "%s" % (self.valPump0),
				(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
			self.pump0.data = True
		else:
			cv2.circle(overlay, (20, 340), 20, (0, 0, 255), -1)
			self.pump0.data = False

		if (self.fingerTipX <= 20 + 20 and self.fingerTipX >= 20 - 20) and \
			(self.fingerTipY <= 80 + 20 and self.fingerTipY >= 80 - 20):
			cv2.circle(overlay, (20, 80), 20, (0, 255, 0), -1)
			cv2.putText(overlay, "%s" % (self.valPump1),
				(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)
			self.pump1.data = True
		else:
			cv2.circle(overlay, (20, 80), 20, (0, 0, 255), -1)
			self.pump1.data = False

#		cv2.circle(overlay, (self.imgWidth-20, 400), 20, (255, 0, 255), -1)
#		cv2.circle(overlay, (self.imgWidth-20, 80), 20, (255, 0, 255), -1)

		cv2.circle(overlay, (self.fingerTipX, self.fingerTipY), 20, (0, 255, 0), -1)
#		cv2.putText(overlay, "%d : %d" % (self.fingerTipX, self.fingerTipY),
#			(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
#			self.color, self.thickness, self.lineType, self.bottomLeftOrigin)

		if (self.fingerTipX <= self.imgWidth//2 + 200 and self.fingerTipX >= self.imgWidth//2 - 200) and \
			(self.fingerTipY <= self.imgHeight//2 + 200 and self.fingerTipY >= self.imgHeight//2 - 200):
			cv2.putText(overlay, "Soils: %.2f" % ((self.soil0 + self.soil1 + self.soil2 + self.soil3) / 4),
				(self.fingerTipX, self.fingerTipY), self.fontFace, self.fontScale,
				self.color, self.thickness, self.lineType, self.bottomLeftOrigin)

		self.pump0_pub.publish(self.pump0.data)
		self.pump1_pub.publish(self.pump1.data)
		self.pump2_pub.publish(self.pump2.data)
		self.pump3_pub.publish(self.pump3.data)
		self.light_pub.publish(self.light.data)
		self.fan_pub.publish(self.fan.data)

		# apply the overlay
		cv2.addWeighted(overlay, self.alpha, self.cv_image, 1 - self.alpha, 0, self.cv_image)
#		print(self.imgHeight)

if __name__ == '__main__':

	# Initialize
	rospy.init_node('camera_preview_macbook_nuc', anonymous=False)
	camera = CameraPreview()

	r = rospy.Rate(10)

	# Camera preview
	while not rospy.is_shutdown():
		camera.cbPreview()
		r.sleep()


import tacticenv

from pyasm.security import *
from pyasm.unittest import UnittestEnvironment, Sample3dEnvironment
from pyasm.common.spt_date import SPTDate

from .sql import *
from .search import *
from .transaction import *
from .database_impl import *
from pyasm.unittest import *
from pyasm.biz import Project
import unittest
from datetime import datetime
from dateutil import parser
import pytz

class SearchTest(unittest.TestCase):

    def _commit(self, impl):
        if impl.commit_on_schema_change():
            DbContainer.commit_thread_sql()

    def test_all(self):
        # start batch environment
        Batch()
        from pyasm.web.web_init import WebInit
        WebInit().execute()

        sample3d_test_env = Sample3dEnvironment()
        sample3d_test_env.create()

        test_env = UnittestEnvironment()
        test_env.create()

        db_res = DbResource.get_default('unittest')
        sql = DbContainer.get(db_res)
        impl = sql.get_database_impl()
        db_type = impl.get_database_type()
        if db_type == "PostgreSQL":
            self.prefix = '''"unittest"."public".'''
            self.sthpw_prefix = '''"sthpw"."public".'''
        elif db_type == "Sqlite":
            self.prefix = ""
            self.sthpw_prefix = ""
        elif db_type == "SQLServer":
            self.prefix = '''"unittest"."dbo".'''
            self.sthpw_prefix = '''"sthpw"."dbo".'''

        else:
            self.prefix = '''"unittest".'''
            self.sthpw_prefix = '''"sthpw".'''

        self.transaction = Transaction.get(create=True)
        try:

            self.person = Person.create( "5", "a",
                    "ComputerWorld", "1")
            self.person = Person.create( "4", "b",
                    "ComputerWorld", "3")
            self.person = Person.create( "3", "c",
                    "ComputerWorld", "3")
            self.person = Person.create( "2", "d",
                    "ComputerWorld", "4")
            self.person = Person.create( "1", "e",
                    "ComputerWorld", "5")

            self._test_no_id()

            self._test_order_by()
            self._test_search_key()
            self._test_search()
            self._test_multi_db_subselect()

            # FIXME: this requires sample3d project
            self._test_search_other_project()
            self._test_search_type()
            self._test_metadata()
            self._test_search_type_existence()
            self._test_project()
            self._test_search_filter()
            self._test_dates_search()
            self._test_child_search()
            self._test_parent_search()
            self._test_add_column_search()
            self._test_commit()
            self._test_set_value()
            self._test_search_set_value()
            self._test_get_by_statement()
            self._test_views()
            self._test_timezone()

        finally:
            self.transaction.rollback()
            Project.set_project('unittest')

            self._commit(impl)

            test_env.delete()
            sample3d_test_env.delete()

    def _test_no_id(self):

        sobject = SearchType.create("unittest/no_id")
        sobject.set_value("name", "Cindy")
        sobject.commit()
        first_id = sobject.get_id()
        self.assertEqual(first_id, sobject.get_value("test_id"))

        sobject = SearchType.create("unittest/no_id")
        sobject.set_value("name", "Mike")
        sobject.commit()
        second_id = sobject.get_id()
        self.assertEqual(second_id, sobject.get_value("test_id"))

        # test update
        sobject.set_value("name", "Michael")
        sobject.commit()

        # test search
        search = Search("unittest/no_id")
        sobjects = search.get_sobjects()
        self.assertEqual( 2, len(sobjects) )
        for sobject in sobjects:
            self.assertEqual( True, sobject.get_id() in [first_id, second_id])

    def _test_order_by(self):

        sobjects = Search.eval("@SOBJECT(unittest/person['@ORDER_BY','description desc, name_first desc'])")
        sobjects1 = Search.eval("@SOBJECT(unittest/person['@ORDER_BY','description, name_first'])")
        sobjects2 = Search.eval("@SOBJECT(unittest/person['@ORDER_BY','description desc, name_first'])")
        name_list = []
        name_list1 = []
        name_list2 = []
        for x in sobjects:
             name = x.get_value('name_last')
             name_list.append(name)
        for x in sobjects1:
             name = x.get_value('name_last')
             name_list1.append(name)
        for x in sobjects2:
             name = x.get_value('name_last')
             name_list2.append(name)
        self.assertEqual(name_list,['e','d','b','c','a'])
        self.assertEqual(name_list1,['a','c','b','d','e'])
        self.assertEqual(name_list2,['e','d','c','b','a'])

        search = Search('unittest/city')
        search.add_order_by('name')
        statement = search.get_statement()
        self.assertEqual(statement, '''SELECT %s"city".* FROM %s"city" ORDER BY "city"."name"''' % (self.prefix, self.prefix))

        search = Search('unittest/person')
        search.add_order_by('unittest/city.unittest/country.code')
        statement = search.get_statement()
        self.assertEqual(statement, '''SELECT %s"person".* FROM %s"person" LEFT OUTER JOIN %s"city" ON "person"."city_code" = "city"."code" LEFT OUTER JOIN %s"country" ON "city"."country_code" = "country"."code" ORDER BY "country"."code"''' % (self.prefix, self.prefix, self.prefix, self.prefix))

        search = Search('unittest/person')
        search.add_order_by('unittest/city.id')
        statement = search.get_statement()
        self.assertEqual(statement, '''SELECT %s"person".* FROM %s"person" LEFT OUTER JOIN %s"city" ON "person"."city_code" = "city"."code" ORDER BY "city"."id"''' % (self.prefix, self.prefix, self.prefix))

        search = Search('unittest/person')
        search.add_order_by('unittest/city.id desc')
        statement = search.get_statement()
        self.assertEqual(statement, '''SELECT %s"person".* FROM %s"person" LEFT OUTER JOIN %s"city" ON "person"."city_code" = "city"."code" ORDER BY "city"."id" desc''' % (self.prefix, self.prefix, self.prefix))

        # with the built-in order-by logic, order by code is added
        search.get_sobjects()
        statement = search.get_statement()
        self.assertEqual(statement, '''SELECT %s"person".* FROM %s"person" LEFT OUTER JOIN %s"city" ON "person"."city_code" = "city"."code" ORDER BY "city"."id" desc, "person"."code"''' % (self.prefix, self.prefix, self.prefix))

    def _test_get_by_statement(self):
        types = ['admin','ben','beth']
        joined_statements = []

        db_res = DbResource.get_default('unittest')
        sql = DbContainer.get(db_res)
        impl = sql.get_database_impl()
        db_type = impl.get_database_type()

        for type in types:
            select = Search('sthpw/login')
            select.add_filter('login', type)
            select.set_show_retired(False)

            if db_type != 'Sqlite':
                select.add_order_by("login")
            statement = select.get_statement()
            joined_statements.append(statement)

        if len(joined_statements) > 1:

            if db_type == 'Sqlite':
                joined_statements = ["%s"%x for x in joined_statements]
            else:
                joined_statements = ["(%s)"%x for x in joined_statements]
            statement = ' union all '.join(joined_statements)
        elif len(joined_statements) == 1:
            statement = joined_statements[0]

        logins =  Login.get_by_statement(statement)

        # This assumes these users actually exist in the database, which they
        # often don't
        #self.assertEqual(len(logins), 3)

        if db_type =='Sqlite':
            self.assertEqual(statement, """SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'admin' union all SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'ben' union all SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'beth'""".format(self.sthpw_prefix))
        else:

            self.assertEqual(statement, '''(SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'admin' ORDER BY "login"."login") union all (SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'ben' ORDER BY "login"."login") union all (SELECT {0}"login".* FROM {0}"login" WHERE "login"."login" = 'beth' ORDER BY "login"."login")'''.format(self.sthpw_prefix))

    def _test_add_column_search(self):
        search = Search('sthpw/task')
        search.add_filter('status','Assignment')
        search.add_column('id')
        search.add_column('timestamp')
        sobject = search.get_sobject()
        self.assertEqual(len(sobject.data), 2)
        self.assertEqual(True, sobject.has_value('timestamp'))

        search = Search('sthpw/task')
        search.add_filter('status','Assignment')
        sobject = search.get_sobject()

        #self.assertEqual(len(sobject.data), 27)
        self.assertEqual(True, sobject.has_value('assigned'))
        self.assertEqual(True, sobject.has_value('process'))

    def _test_child_search(self):

        from pyasm.biz import Task
        person = SearchType.create('unittest/person')
        person.set_value('name_first','pete')
        person.commit()
        for xrange in (1, 50):
            Task.create(person, 'process_AA','some task', 'admin', context='process_AA')
        person2 = SearchType.create('unittest/person')
        person2.set_value('name_first','jamie')
        person2.commit()

        person2_tasks = []
        for xrange in (1, 50):
            person2_tasks.append(Task.create(person2, 'process_BB','some task', 'admin', context='process_BB'))
        task_search = Search('sthpw/task')
        task_search.add_filters('process', ['process_AA', 'process_BB'])
        tasks = task_search.get_sobjects()
        search2 = Search('unittest/person')
        search2.add_relationship_filters(tasks)
        persons = search2.get_sobjects()

        search3 = Search('unittest/person')
        search3.add_relationship_search_filter(task_search)
        persons_fast = search3.get_sobjects()

        self.assertEqual(SObject.get_values(persons, 'id'), SObject.get_values(persons_fast, 'id'))
        self.assertEqual(SObject.get_values(persons_fast, 'name_first'), ['pete','jamie'])

        # if I retire all the tasks for person2
        for task in tasks:
            if task.get_value('process') =='process_BB':
                task.retire()

        task_search = Search('sthpw/task')
        task_search.add_filters('process', ['process_AA', 'process_BB'])
        tasks = task_search.get_sobjects()
        search4 = Search('unittest/person')
        search4.add_relationship_search_filter(task_search)
        persons_fast = search4.get_sobjects()

        search2 = Search('unittest/person')
        search2.add_relationship_filters(tasks)
        persons = search2.get_sobjects()

        self.assertEqual(SObject.get_values(persons, 'id'), SObject.get_values(persons_fast, 'id'))
        self.assertEqual(SObject.get_values(persons_fast, 'name_first'), ['pete'])

        # test add_filters() with an empty array
        task_search = Search('sthpw/task')
        task_search.add_filters('process', [])
        tasks = task_search.get_sobjects()
        self.assertEqual(tasks, [])
        expected = '''SELECT %s"task".* FROM %s"task" WHERE "task"."id" is NULL AND ("task"."s_status" != 'retired' or "task"."s_status" is NULL) ORDER BY "task"."search_type", "task"."search_code"'''%(self.sthpw_prefix, self.sthpw_prefix)
        statement = task_search.get_statement()
        self.assertEqual(statement, expected)

    def _test_parent_search(self):
        from pyasm.biz import Task
        person = SearchType.create('unittest/person')
        person.set_value('name_first','burt')
        person.commit()

        person2 = SearchType.create('unittest/person')
        person2.set_value('name_first','sean')
        person2.commit()

        person_search = Search('unittest/person')
        person_search.add_filters('id', [person.get_id(), person2.get_id()])
        for xrange in (1, 50):
            Task.create(person, 'process_CC','some task', 'admin', context='process_CC')
        for xrange in (1, 50):
            Task.create(person2, 'process_DD','some task', 'admin')

        # find parent of tasks
        search2 = Search('sthpw/task')
        search2.add_relationship_filters([person,person2])
        tasks = search2.get_sobjects()

        search3 = Search('sthpw/task')
        search3.add_relationship_search_filter(person_search)
        tasks2 = search3.get_sobjects()

        self.assertEqual(SObject.get_values(tasks, 'id'), SObject.get_values(tasks2, 'id'))

    def _test_search_other_project(self):
        # while in unittest, search for pacman assets
        search = Search('sthpw/search_object')
        search.add_filter('search_type','prod/asset')
        sobject = search.get_sobject()
        # make sure the database field is not hardcoded pacman
        sobject.set_value('database', '{project}')
        sobject.commit()

        asset = SearchType.create('prod/asset?project=sample3d')
        asset.set_value('code','unittest001')
        asset.set_value('pipeline_code','model')
        asset.commit()

        asset = SearchType.create('prod/asset?project=sample3d')
        asset.set_value('code','unittest002')
        asset.set_value('pipeline_code','model')
        asset.commit()

        search2 = Search("prod/asset?project=sample3d")
        search2.add_filter("pipeline_code", "model")
        sobjects = search2.get_sobjects()
        self.assertEqual(True, len(sobjects) > 0 )
        self.assertEqual("model", sobjects[0].get_value('pipeline_code'))
        self.assertEqual("sample3d", sobjects[0].get_database())
        self.assertEqual("sample3d", sobjects[1].get_database())

        search3 = Search("prod/asset?project=sample3d")
        search3.add_regex_filter("pipeline_code", "model|cg_asset")
        search3.add_filter("code", "chr003")
        sobject = search3.get_sobject()
        if sobject:
            sobject.set_value('description','some char')
            sobject.commit()

            updated_sobject = search3.get_sobject(redo=True)
            self.assertEqual("some char", updated_sobject.get_value('description'))

        search4 = Search("prod/asset?project=sample3d")
        search4.add_regex_filter("pipeline_code", "cg_asset's")

        sobject = search4.get_sobject()
        self.assertEqual(sobject, None)

    def _test_search(self):

        # create the person
        person = SObjectFactory.create("unittest/person")
        person.set_value("name_first", "XXX")
        person.set_value("name_last", "Smith")
        person.set_value("nationality", "Canada")
        person.commit()

        search = Search("unittest/person")
        search.add_filter("name_first", "XXX")
        search.add_filter("name_last", "Smith")
        person = search.get_sobject()
        self.assertEqual("XXX", person.get_value("name_first") )

        xxx_id = person.get_id()

        # test_set_value
        person.set_value("name_first", "YYY")
        person.set_value("name_last", "Cowser")
        person.commit()

        search2 = Search("unittest/person")
        search2.add_filter("name_first", "YYY")
        search2.add_filter("name_last", "Cowser")
        person2 = search2.get_sobject()

        yyy_id = person2.get_id()

        self.assertEqual( xxx_id, yyy_id )

        # test attributes
        name_first = person.get_attr_value("name_first")
        self.assertEqual("YYY", name_first )

        name_last = person.get_attr_value("name_last")
        self.assertEqual("Cowser", name_last )

        # test related search
        sto_search = Search('prod/shot?project=sample3d')
        sobjects = sto_search.get_sobjects()
        self.assertEqual(len(sobjects) > 1, True)

        search = Search('sthpw/login')

        # this should not cause errors if there is schema connection between storyboard and login
        login_sobjects = search.add_relationship_filters(sobjects)
        if sobjects:
            login_sobject = search.add_relationship_filter(sobjects[0])

    def _test_search_type(self):
        '''test that search types behave properly for templating'''
        SearchType.set_global_template("project", "unittest")

        #search_type = SearchType.get("prod/asset")
        sobject = SearchType.create("unittest/person")
        key = sobject.get_search_type()
        self.assertEqual(key, "unittest/person?project=unittest")

    def _test_metadata(self):
        # create an arbitrary sobject
        person = SearchType.create("unittest/person")

        # get metadata that does not exist
        cow = person.get_metadata_value("cow")
        self.assertEqual("", cow)

        # json dict doesn't have this method
        #person.get_metadata_xml().clear_xpath_cache()

        # set the value
        person.set_metadata_value("cow", "angus")
        cow = person.get_metadata_value("cow")
        self.assertEqual("angus", cow)

        # add a whole bunch of entries:
        data = {
            'color': 'blue',
            'height': '195cm',
            'weight': '102Kg'
        }
        person.add_metadata(data)

        color = person.get_metadata_value("color")
        height = person.get_metadata_value("height")
        self.assertEqual("blue", color)
        self.assertEqual("195cm", height)

        # replace the entire structure
        person.add_metadata(data, replace=True)
        cow = person.get_metadata_value("cow")
        self.assertEqual("", cow)

        # if someone set it with non-json data, it should still return as is
        person.metadata = None
        person.set_value('metadata','testing==OK')
        meta_dict = person.get_metadata_dict()
        self.assertEqual('testing==OK', meta_dict)

    def _test_search_type_existence(self):
        '''test a bunch of search types to see if they are registered'''
        # DEPRECATED
        return
        #TODO: add more search types here
        st_list = ['prod/shot_instance','prod/sequence_instance','sthpw/widget_extend']
        for st in st_list:
            search = Search(SearchType.SEARCH_TYPE)
            search.add_filter('search_type', st)
            sobjects = search.get_sobjects()
            self.assertEqual(len(sobjects), 1)

    def _test_search_key(self):

        # test a correct search_key
        search_key = "prod/asset?project=sample3d&code=chr001"

        search_type = SearchKey.extract_base_search_type(search_key)
        self.assertEqual("prod/asset", search_type)

        project = SearchKey.extract_project(search_key)
        self.assertEqual("sample3d", project)

        code = SearchKey.extract_code(search_key)
        self.assertEqual("chr001", code)

        # test a long one
        search_key = "prod/asset?project=sample3d&db=postgres&code=chr001"

        search_type = SearchKey.extract_base_search_type(search_key)
        self.assertEqual("prod/asset", search_type)

        database = SearchKey.extract_database(search_key)
        self.assertEqual("postgres", database)

        project = SearchKey.extract_project(search_key)
        self.assertEqual("sample3d", project)

        code = SearchKey.extract_code(search_key)
        self.assertEqual("chr001", code)

        #eval(@GET(prod/asset.code))
        #eval(@GET(prod/asset.code?project=sample3d)) ???

        search_key = "sthpw/project"
        code = "unittest"

        search = Search(search_key)
        search.add_filter("code", code)
        sobject = search.get_sobject()

        search_key = SearchKey.get_by_sobject(sobject)
        self.assertEqual("sthpw/project?code=unittest", search_key)

    def _test_project(self):

        from pyasm.biz import Project
        sql = Project.get_database_impl()

        # Don't bother running if you don't have sample3d
        if not sql.database_exists('sample3d'):
            return

        Project.set_project("sthpw")

        sobject = SearchType.create('prod/shot?project=sample3d', columns=['code', 'sequence_code', 'pipeline_code'], result=['S001','HT001','shot'])
        self.assertEqual("prod/shot?project=sample3d", sobject.get_search_type())

        if sql.database_exists('sample3d'):
            db_resource = Project.get_db_resource_by_search_type('prod/bin?project=sample3d')
            exists= sql.table_exists(  db_resource ,'bin')
            if exists:
                search = Search('prod/bin', project_code='sample3d')
                self.assertEqual("prod/bin?project=sample3d", search.get_search_type())
        # check that a search type is properly created
        search_type = SearchType.get("prod/shot?project=sample3d")
        base_key = search_type.get_base_key()
        self.assertEqual("prod/shot", base_key)

        # NOTE: search_type get_full_key() method is deprecated.

        # test that the sobject maintains the search type
        sobject = SearchType.create("prod/shot?project=sample3d")
        search_type = sobject.get_search_type()
        self.assertEqual("prod/shot?project=sample3d", search_type)

        # set it back to unittest
        Project.set_project("unittest")

        # test current project is added when there is not project set
        sobject = SearchType.create("prod/shot")
        search_type = sobject.get_search_type()
        self.assertEqual("prod/shot?project=unittest", search_type)

        # test current project is added when there is not project set, even
        # when the project has changed
        sobject = SearchType.create("prod/shot")

        search_type = sobject.get_search_type()
        self.assertEqual("prod/shot?project=unittest", search_type)

        if sql.database_exists('sample3d'):
            Project.set_project("sample3d")

            project_code = Project.get_project_code()
            self.assertEqual("sample3d", project_code)

        # set it back to unittest project
        Project.set_project("unittest")

        # test the search
        if sql.database_exists('sample3d'):
            search_type = "prod/shot?project=sample3d"
            search = Search(search_type)
            project_code = search.get_project_code()
            self.assertEqual("sample3d", project_code)

            # test the search project code even though the project has hanved
            search_type = "prod/shot?project=sample3d"
            search = Search(search_type)
            project_code = search.get_project_code()
            self.assertEqual("sample3d", project_code)

            Project.set_project("admin")
            project_code = search.get_project_code()
            self.assertEqual("sample3d", project_code)

            project_code = Project.get_project_code()
            self.assertEqual("admin", project_code)

        # set it back to unittest project
        Project.set_project("unittest")

    def _test_search_filter(self):
        search = Search("unittest/person")
        search.add_filter("name_first", "YYY")
        search.add_filter("name_last", "Cowser")

        statement = search.get_statement()
        expected = """SELECT %s"person".* FROM %s"person" WHERE "person"."name_first" = 'YYY' AND "person"."name_last" = 'Cowser'""" % (self.prefix, self.prefix)

        self.assertEqual(expected, statement)

        search2 = Search("unittest/city")
        search2.add_search_filter("id", search)

        statement = search2.get_statement()
        expected = """SELECT {0}"city".* FROM {0}"city" WHERE "city"."id" in ( SELECT {0}"person".* FROM {0}"person" WHERE "person"."name_first" = 'YYY' AND "person"."name_last" = 'Cowser' )""".format(self.prefix)

        self.assertEqual(expected, statement)

        search = Search("unittest/person")
        search.add_op('begin')
        search.add_filter("name_first", "YYY")
        search.add_filter("name_last", "Cowser")
        search.add_op('or')
        search.add_filter("city_code", "YYZ")
        statement = search.get_statement()
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE ( "person"."name_first" = 'YYY' OR "person"."name_last" = 'Cowser' ) AND "person"."city_code" = 'YYZ'""".format(self.prefix)

        self.assertEqual(expected, statement)

        # mix "is not" with "is" logic
        search = Search("unittest/person")

        search.add_op('begin')
        search.add_filter('name_first', 'T', op='~')
        search.add_filter('name_first', 'U', op='~')
        search.add_op('or')

        search.add_op('begin')
        search.add_filter('nationality','canadian', op='!=')
        search.add_filter('nationality', None)
        search.add_op('or')

        search.add_op('begin')
        search.add_filter("name_first", "YYY")
        search.add_filter("name_last", "Cowser")
        search.add_filter("city_code", "YYZ")
        search.add_op('or')

        # this is optional
        #search.add_op('and')
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE ( "person"."name_first" ~ 'T' OR "person"."name_first" ~ 'U' ) AND ( "person"."nationality" != 'canadian' OR "person"."nationality" is NULL ) AND ( "person"."name_first" = 'YYY' OR "person"."name_last" = 'Cowser' OR "person"."city_code" = 'YYZ' )""".format(self.prefix)

        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # old style without begin
        search = Search('sthpw/login')
        search.add_where("\"login\".\"license_type\" = 'user'")
        search.add_where("\"login\".\"license_type\" is NULL")
        search.add_op('or')

        expected = """SELECT {0}"login".* FROM {0}"login" WHERE "login"."license_type" = \'user\' OR "login"."license_type" is NULL""".format(self.sthpw_prefix)
        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # extra dangling begin, default back to AND
        search = Search('sthpw/login')
        #search.add_op('begin')
        search.add_op('begin')
        search.add_where("\"login\".\"license_type\" = 'user'")
        search.add_where("\"login\".\"license_type\" is NULL")
        #search.add_op('or')

        expected = """SELECT {0}"login".* FROM {0}"login" WHERE "login"."license_type" = \'user\' AND "login"."license_type" is NULL""".format(self.sthpw_prefix)
        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # extra dangling begin, with 2 ORs applied already
        search = Search('sthpw/login')
        search.add_op('begin')
        search.add_op('begin')
        search.add_where("\"login\".\"license_type\" = 'user'")
        search.add_where("\"login\".\"license_type\" is NULL")
        search.add_op('or')

        search.add_op('begin')
        search.add_where("\"login\".\"license_type\" = 'float'")
        search.add_op('or')

        expected = """SELECT {0}"login".* FROM {0}"login" WHERE ( "login"."license_type" = \'user\' OR "login"."license_type" is NULL ) AND "login"."license_type" = \'float\'""".format(self.sthpw_prefix)
        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # extra dangling begin, with 2 ORs applied already
        search = Search('sthpw/login')
        search.add_op('begin')
        search.add_filter('namespace','sample3d')
        search.add_op('begin')
        search.add_where("\"login\".\"license_type\" = 'user'")
        search.add_where("\"login\".\"license_type\" is NULL")
        search.add_op('or')

        search.add_op('begin')
        search.add_where("\"login\".\"license_type\" = 'float'")
        search.add_op('or')
        search.add_op('or')

        expected = """SELECT {0}"login".* FROM {0}"login" WHERE "login"."namespace" = \'sample3d\' OR ( "login"."license_type" = \'user\' OR "login"."license_type" is NULL ) OR "login"."license_type" = \'float\'""".format(self.sthpw_prefix)
        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # triple ands on level 0, OR on level 1
        search = Search("sthpw/task")

        search.add_op('begin')

        search.add_op('begin')
        search.add_filter('process', 'anim')
        search.add_filter('status', 'Pending')
        search.add_filter('assigned' , 'admin')
        search.add_op('and')

        search.add_op('begin')
        search.add_filter('process', 'layout')
        search.add_filter('status', 'Assignment')
        search.add_filter('assigned' , 'fil')
        search.add_op('and')

        search.add_op('or')

        expected = """SELECT {0}"task".* FROM {0}"task" WHERE ( "task"."process" = \'anim\' AND "task"."status" = \'Pending\' AND "task"."assigned" = \'admin\' ) OR ( "task"."process" = \'layout\' AND "task"."status" = \'Assignment\' AND "task"."assigned" = \'fil\' )""".format(self.sthpw_prefix)

        statement = search.get_statement()
        self.assertEqual(expected, statement)

        # test is NULL and is not NULL
        search = Search("unittest/person")
        search.add_op_filters([("name_first", 'is', 'NULL')])

        statement = search.get_statement()
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE "person"."name_first" is NULL""".format(self.prefix)
        self.assertEqual(expected, statement)

        # lowercase null treated as string
        search = Search("unittest/person")
        search.add_op_filters([("name_first", 'is', 'null')])

        statement = search.get_statement()
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE "person"."name_first" is 'null'""".format(self.prefix)
        self.assertEqual(expected, statement)

        search = Search("unittest/person")
        search.add_op_filters([("name_last", 'is not', 'NULL')])

        statement = search.get_statement()
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE "person"."name_last" is not NULL""".format(self.prefix)
        self.assertEqual(expected, statement)

    def _test_relationship_filter(self):

        search = Search("unittest/person")
        search.add_filter("name_first", "YYY")
        search.add_filter("name_last", "Cowser")

        search2 = Search("unittest/city")
        search2.add_search_filter("id", search)

    def _test_dates_search(self):
        import time
        timezone = time.tzname[time.daylight]

        from pyasm.biz import Project
        database_type = Project.get_by_code("unittest").get_database_type()
        start_range = SPTDate.convert_to_timezone("2010-01-01", timezone)
        end_range = SPTDate.convert_to_timezone("2010-02-02", timezone)

        if database_type == "MySQL":
            start_range = start_range.replace(tzinfo=None)
            end_range = end_range.replace(tzinfo=None)

        search = Search("unittest/person")
        search.add_date_range_filter("birth_date", "2010-01-01", "2010-02-01")
        expected = """SELECT {0}"person".* FROM {0}"person" WHERE "person"."birth_date" >= '{1}' AND "person"."birth_date" < '{2}'""".format(self.prefix, start_range, end_range)
        self.assertEqual(expected, search.get_statement() )

        search = Search("unittest/person")
        search.add_dates_overlap_filter("birth_date", "birth_date", "2010-01-01", "2010-02-01")
        expected = '''SELECT {0}"person".* FROM {0}"person" WHERE "person"."id" in (SELECT {0}"person"."id" FROM {0}"person" '''.format(self.prefix)
        expected += '''WHERE ( "person"."birth_date" <= '{0}' AND "person"."birth_date" >= '{0}' ) '''.format(start_range)
        expected += '''OR ( "person"."birth_date" >= '{0}' AND "person"."birth_date" <= '{0}' ) '''.format(end_range)
        expected += '''OR ( "person"."birth_date" >= '{0}' AND "person"."birth_date" <= '{1}' ))'''.format(start_range, end_range)
        self.assertEqual(expected, search.get_statement() )

    def _test_commit(self):

        from pyasm.biz import Project
        database_type = Project.get_by_code("unittest").get_database_type()
        if database_type == "MySQL":
            print
            print("WARNING: !!!!!!!")
            print("_test_commit is disabled")
            print("WARNING: !!!!!!!")
            print
            return

        person_s = Search('unittest/person')
        person_s.add_filter('name_first','pete')

        person = person_s.get_sobject()
        from pyasm.biz import Note
        Note.create(person, "3 slashes \\\\\\", context="unittest_commit", process="unittest_commit")
        search = Search('sthpw/note')
        search.add_filter('process','unittest_commit')
        search.set_limit(1)
        note = search.get_sobject()
        self.assertEqual(note.get_value('note'), "3 slashes \\\\\\")

    def _test_set_value(self):
        ''' test with different Database Impl'''
        update = Update()
        update.set_database('sthpw')
        update.set_table('task')
        update.set_value('timestamp','2012-12-12')
        self.assertEqual( update.get_statement(), """UPDATE {0}"task" SET "timestamp" = '2012-12-12'""".format(self.sthpw_prefix))

        update = Update()
        update.set_database('sthpw')
        update.set_table('task')

        # Changing database to SQLServer
        sql_impl = DatabaseImpl.get('SQLServer')

        update.impl = sql_impl
        value = '2012-12-12'
        value = update.impl.process_date(value)

        update.set_value('timestamp', value)
        self.assertEqual( update.get_statement(), """UPDATE {0}"task" SET "timestamp" = convert(datetime2, \'2012-12-12\', 0)""".format(self.sthpw_prefix))
        update.set_value('timestamp','NOW')
        self.assertEqual( update.get_statement(), """UPDATE {0}"task" SET "timestamp" = getdate()""".format(self.sthpw_prefix))

        from pyasm.biz import Project
        database_type = Project.get_by_code("unittest").get_database_type()

        # time_dict = {'SQLServer': "convert(datetime2, '2012-12-25', 0)",
        #             'Sqlite':"'2012-12-25'",
        #             'PostgreSQL':"'2012-12-25'",
        #             'MySQL': "'2012-12-25 00:00:00'"}
        #             #'Oracle':"TO_DATE('2012-12-25','YYYY-MM-DD'"}
        # #TODO: test with cx_Oracle installed
        # #TODO: have another test with timezone considered
        # for db_type in ['Sqlite','SQLServer','MySQL','PostgreSQL']:
        #     sql_impl = DatabaseImpl.get(db_type)
        #     update = Update()
        #     update.set_database('sthpw')
        #     update.set_table('task')
        #     update.impl = sql_impl
        #     value = '2012-12-25'
        #     value = update.impl.process_date(value)
        #     update.set_value('timestamp', value)
        #     update.set_value('description','')
        #     if db_type == 'SQLServer':
        #         self.assertEqual( update.get_statement(), """UPDATE %s"task" SET "timestamp" = %s, "description" = N\'\'"""% (self.sthpw_prefix, time_dict.get(db_type)))
        #     else:
        #         self.assertEqual( update.get_statement(), """UPDATE %s"task" SET "timestamp" = %s, "description" = \'\'"""% (self.sthpw_prefix, time_dict.get(db_type)))

        #     update.set_value('description',None)
        #     self.assertEqual( update.get_statement(), """UPDATE %s"task" SET "timestamp" = %s, "description" = NULL"""% (self.sthpw_prefix, time_dict.get(db_type)))

    def _test_timezone(self):

        # timestamp without timezone: set value without timezone conversion

        task = SearchType.create('sthpw/task')
        input_time = datetime.utcnow().replace(microsecond=0)

        task.set_value('timestamp', input_time)
        task.set_value('description', 'search_test')
        output_time = task.get_datetime_value('timestamp')
        self.assertEqual(input_time, output_time)
        task.commit()
        output_time = task.get_datetime_value('timestamp')
        self.assertEqual(input_time, output_time)

        # timestamp without timezone: set value with timezone conversion

        task1 = SearchType.create('sthpw/task')
        input_time1 = input_time
        local_tz = pytz.timezone("Europe/Moscow")
        tz_input_time = input_time1.replace(tzinfo=pytz.utc).astimezone(local_tz)

        task1.set_value('timestamp', tz_input_time)
        output_time = task1.get_datetime_value('timestamp')
        self.assertEqual(SPTDate.convert_to_timezone(tz_input_time, 'UTC'), output_time)
        task1.commit()
        output_time = task1.get_datetime_value('timestamp')
        self.assertEqual(input_time1, output_time)

	    # search for task using both timestamps

        search = Search('sthpw/task')
        search.add_filter('description', 'search_test')
        search.add_filter('timestamp', input_time)
        search_result = search.get_sobject()
        self.assertEqual(task.get_code(), search_result.get_code())

        search = Search('sthpw/task')
        search.add_filter('description', 'search_test')
        search.add_filter('timestamp', tz_input_time)
        search_result = search.get_sobject()
        self.assertEqual(task.get_code(), search_result.get_code())

        # timestamp with timezone: set value without timezone conversion

        ticket = SearchType.create('sthpw/ticket')
        input_time = datetime.utcnow().replace(microsecond=0)

        ticket.set_value('timestamp', input_time)
        ticket.set_value('login', 'admin')
        ticket.set_value('ticket', 'search_test')
        ticket.set_value('expiry', input_time)
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time, output_time)
        ticket.commit()
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time, output_time)

        # timestamp with timezone: set value with timezone conversion

        ticket1 = SearchType.create('sthpw/ticket')
        input_time1 = input_time
        local_tz = pytz.timezone("Europe/Moscow")
        tz_input_time = input_time1.replace(tzinfo=pytz.utc).astimezone(local_tz)

        ticket1.set_value('timestamp', tz_input_time)
        ticket1.set_value('login', 'admin')
        ticket1.set_value('ticket', 'test2')
        ticket1.set_value('expiry', tz_input_time)
        output_time = ticket1.get_datetime_value('timestamp')
        self.assertEqual(SPTDate.convert_to_timezone(tz_input_time, 'UTC'), output_time)
        ticket1.commit()
        output_time = ticket1.get_datetime_value('timestamp')
        self.assertEqual(input_time1, output_time)

	    # search ticket using both timestamps

        search = Search('sthpw/ticket')
        search.add_filter('ticket', 'search_test')
        search.add_filter('timestamp', input_time)
        search_result = search.get_sobject()
        self.assertEqual(ticket.get_code(), search_result.get_code())

        search = Search('sthpw/ticket')
        search.add_filter('ticket', 'search_test')
        search.add_filter('timestamp', tz_input_time)
        search_result = search.get_sobject()
        self.assertEqual(ticket.get_code(), search_result.get_code())

        # test set_value, get_value with string date

        input_time_string = "2019-07-02 18:17:42"
        input_time_date = parser.parse(input_time_string)

        ticket = SearchType.create('sthpw/ticket')
        ticket.set_value('timestamp', input_time_string)
        ticket.set_value('login', 'admin')
        ticket.set_value('ticket', 'set_test')
        ticket.set_value('expiry', input_time_string)
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time_date, output_time)
        ticket.commit()
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time_date, output_time)

        #Test set_value, get_value with timezoned string date

        input_time_string_tz = "2019-07-02 18:17:42+03:00"
        input_time_date_tz = parser.parse(input_time_string_tz)
        input_time_date_utc = SPTDate.convert_to_timezone(input_time_date_tz, 'UTC').replace(tzinfo=None)

        ticket = SearchType.create('sthpw/ticket')
        ticket.set_value('timestamp', input_time_string_tz)
        ticket.set_value('login', 'admin')
        ticket.set_value('ticket', 'set_test_tz')
        ticket.set_value('expiry', input_time_string_tz)
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time_date_tz, output_time)
        ticket.commit()
        output_time = ticket.get_datetime_value('timestamp')
        self.assertEqual(input_time_date_utc, output_time)

    def _test_multi_db_subselect(self):

        from pyasm.biz import Task
        person = SearchType.create('unittest/person')
        person.set_value('name_first','carin')
        person.commit()
        Task.create(person, 'subselect','some task', 'admin', status="a")

        from pyasm.biz import Task
        person = SearchType.create('unittest/person')
        person.set_value('name_first','carin2')
        person.commit()
        Task.create(person, 'subselect','some task', 'admin', status="b")

        from pyasm.biz import Task
        person = SearchType.create('unittest/person')
        person.set_value('name_first','carin3')
        person.commit()
        Task.create(person, 'subselect','some task', 'admin', status="c")

        # find people by carin that have a task "subselect"
        search_person = Search("unittest/person")
        search_task = Search("sthpw/task")
        search_person.add_relationship_search_filter(search_task,use_multidb=False)
        search_task.add_filter("process", "subselect")

        sobjects = search_person.get_sobjects()
        self.assertEqual(len(sobjects), 3)

        # find people by carin that have a task "subselect", using
        # subselect
        search_person = Search("unittest/person")
        search_task = Search("sthpw/task")
        search_person.add_relationship_search_filter(search_task,use_multidb=True)
        search_task.add_filter("process", "subselect")

        sobjects = search_person.get_sobjects()
        self.assertEqual(len(sobjects), 3)

        # find task that carin has
        search_task = Search("sthpw/task")
        search_person = Search("unittest/person")
        search_person.add_filter('name_first','carin')
        search_task.add_relationship_search_filter(search_person,use_multidb=True)

        sobjects = search_task.get_sobjects()

        # test an order by on a task status (cross database join)
        # Note this will not work on Postgres or SQLite which do not
        # support cross database joins
        search_person = Search("unittest/person")
        search_person.add_order_by("sthpw/task.status", direction="desc")
        statement = search_person.get_statement()

        can_join = DatabaseImpl.can_search_types_join( \
                "unittest/person", "sthpw/task")

        if can_join:
            expected = '''SELECT %s"person".* FROM %s"person" LEFT OUTER JOIN %s"task" ON "person"."code" = "task"."search_code" WHERE "task"."search_type" = 'unittest/person?project=unittest' ORDER BY "task"."status" desc''' % (self.prefix, self.prefix, self.sthpw_prefix)
            self.assertEqual(expected, statement)

            sobjects = search_person.get_sobjects()
            names = [x.get_value("name_first") for x in sobjects]
            expected = ['carin3','carint2','carin']
            self.assertEqual(expected, names)

    def _test_search_set_value(self):
        #TODO, test other db impl
        sobject = Search.eval("@SOBJECT(sthpw/task['@LIMIT','1'])", single=True)
        sobject.set_value('timestamp','2012-12-12')
        sobject.set_value('description', '')
        sobject.commit(triggers=False)
        self.assertEqual( sobject.data.get('description'), None)
        self.assertEqual( sobject.get_value('description'), '')

    def _test_views(self):

        car = SearchType.create('unittest/car')
        car.set_value('model','Nissan Sentra')
        car.set_value('class','sedan')
        car.commit()

        car = SearchType.create('unittest/car')
        car.set_value('model','Ferrari 958')
        car.set_value('class','sports_car')
        car.commit()

        car = SearchType.create('unittest/car')
        car.set_value('model','Porsche 911')
        car.set_value('class','sports_car')
        car.commit()

        search = Search("unittest/car")
        cars = search.get_sobjects()

        search = Search("unittest/sports_car")
        sports_cars = search.get_sobjects()

        self.assertEqual(3, len(cars))
        self.assertEqual(2, len(sports_cars))

        #for sports_car in sports_cars:
        #    sports_car.set_value("top_speed", "200mph")
        #    sports_car.commit()

if __name__ == "__main__":
    unittest.main()


from utils.resnet_model import get_model
from utils.dataprocessing import chunk_generator, get_chunked_generator
from utils.custom_callbacks import GarbageCollector, custom_mse

import tensorflow as tf

from glob import glob
import numpy as np
np.random.seed(0)
import argparse
from pathlib import Path
import json

gpus = tf.config.experimental.list_physical_devices("GPU")

if gpus:
    for gpu in gpus:
        tf.config.experimental.set_memory_growth(gpu, True)

parser = argparse.ArgumentParser(description='Train model with specific arguments from the grid')
parser.add_argument('--json_arguments', type=str, required=True,
                    help='json file with specified model`s parameters')
args = parser.parse_args()

with open(args.json_arguments, 'rt') as f:
    args = json.load(f)

N = args['N']
BATCH_SIZE = 128
NAME = args['model_name']
THRESHOLD = args['thr']
MOL_DIR = Path(args['directory'])
N_epochs = args['epochs']
SCAFFOLD = args['scaffold']
try: #cubic augmentation randomly choose from 24 possible cubic orientations
    CUBIC_AUGMENTATION = args['cubic']
except:
    CUBIC_AUGMENTATION = True

###data preprocessing
train_files = list(map(str, list(MOL_DIR.joinpath("train").glob("*.npy"))))
test_files = list(map(str, list(MOL_DIR.joinpath("test").glob("*.npy"))))

print("train chunks: ", len(train_files))
print("test chunks: ", len(test_files))

steps_per_epoch = 0
for f in train_files:
    steps_per_epoch += len(np.load(f, 'r'))
steps_per_epoch = int(np.ceil(steps_per_epoch * N / BATCH_SIZE))
val_steps = 0
for f in test_files:
    val_steps += len(np.load(f, 'r'))
val_steps = int(np.ceil(val_steps / BATCH_SIZE))
del f
###generators
train_generator = get_chunked_generator(train_files, batch_size=BATCH_SIZE, N=N, cubic_rotations=CUBIC_AUGMENTATION,)
test_generator = get_chunked_generator(test_files, batch_size=BATCH_SIZE, N=1, cubic_rotations=False,)

###define loss function
if THRESHOLD:
    print("use custom loss")
    loss = custom_mse(thr=THRESHOLD)
else:
    loss = tf.keras.losses.MeanSquaredError()

###model
model = get_model()
model._average_over_rotations = False
model.compile(
    optimizer=tf.keras.optimizers.RMSprop(0.01),  # Optimizer
    # Loss function to minimize
    loss=loss,
    # List of metrics to monitor
    metrics=[tf.keras.metrics.MeanAbsoluteError(), tf.keras.metrics.MeanAbsolutePercentageError()],
)

###callbacks
script_path = Path(__file__).resolve().parent
subgrid_name = str(MOL_DIR).split("/")[-1]
if SCAFFOLD:
    subgrid_name += "_scaffold"
log_output = script_path.joinpath("supp/logs/{}".format(subgrid_name)).joinpath(NAME + ".csv")
log_output.resolve().parent.mkdir(parents=True, exist_ok=True)
log_output.touch(exist_ok=True)

lr_reduce = tf.keras.callbacks.ReduceLROnPlateau(
    monitor='mean_absolute_percentage_error', factor=0.2, patience=int(16 / N), verbose=0, mode='auto',
    min_delta=0.5, cooldown=10, min_lr=1e-4
)
csv_logger = tf.keras.callbacks.CSVLogger(log_output, separator=',', append=False)
garbage_collector = GarbageCollector()

checkpoint_filepath = script_path.joinpath("supp/checkpoints/{}".format(subgrid_name)).joinpath(NAME)
checkpoint_filepath.mkdir(parents=True, exist_ok=True)
print('CHECKPOINT PATH: ', checkpoint_filepath)

checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=True,
    monitor='val_mean_absolute_error',
    mode='min',
    save_best_only=True)

#fit
model.fit(train_generator, epochs=N_epochs, steps_per_epoch=steps_per_epoch,\
    validation_data=test_generator, validation_steps=val_steps,
    callbacks=[lr_reduce, csv_logger, garbage_collector, checkpoint_callback],
    shuffle=False,
    workers=4,
    use_multiprocessing=True,
    max_queue_size=20,
    )


from sdk.common.exceptions.exceptions import ErrorCodes, DetailedException

class PublisherErrorCodes(ErrorCodes):
    DUPLICATE_PUBLISHER_NAME = 1100001

class DuplicatePublisherName(DetailedException):
    def __init__(self, message=False):
        super().__init__(
            code=PublisherErrorCodes.DUPLICATE_PUBLISHER_NAME,
            debug_message=message or "Publisher with that name already exists.",
            status_code=400,
        )


##  aaaabbbbcccaaa >> a4b4c3a3으로 압축
# n시간동안 해결완료
def solution(a):
    tar = a[0]
    cnt = 0
    answer = ""
    for idx in a:
        if idx == tar:
           cnt += 1
           tar = idx
        else:
            answer += (tar + str(cnt))
            cnt = 1
            tar = idx
    answer += (tar + str(cnt))
    print(answer)

solution("aaaabbbbcccaaa")


from collections import defaultdict

class Solution(object):
    def findItinerary(self, tickets):
        """
        :type tickets: List[List[str]]
        :rtype: List[str]
        """
        current = "JFK"
        route = [current]
        return self.findItineraryHelper(tickets, current, route)

    def findItineraryHelper(self, tickets, current, route):
        if len(tickets) == 0:
            return route

        for i, (origin, destination) in enumerate(tickets):
            if origin == current:
                newTicket = tickets[:i] + tickets[i+1:]
                current = destination
                route.append(current)
                return self.findItineraryHelper(newTicket, current, route)
        return None

    def findItineraryHelper2(self, tickets):
        targets = defaultdict(list)
        for a, b in sorted(tickets)[::-1]:
            targets[a].append(b)
        route, stack = [], ["JFK"]

        print(targets)
        while stack:
            print(stack[-1])
            while targets[stack[-1]]:
                l = targets[stack[-1]].pop()
                stack.append(l) #targets[stack[-1]].pop(),
            top = stack.pop()
            route.append(top)
        return route[::-1]

        # while stack:
        #     while targets[stack[-1]]:
        #         stack += targets[targets[-1]].pop()
        #     route.append(stack.pop())
        # return route[::-1]

#tickets = [["MUC", "LHR"], ["JFK", "MUC"], ["SFO", "SJC"], ["LHR", "SFO"]]
tickets = [["JFK","SFO"],["JFK","ATL"],["SFO","ATL"],["ATL","JFK"],["ATL","SFO"]]
s = Solution()
print(s.findItineraryHelper2(tickets))


from __future__ import print_function
import argparse
import time
import os
import sys
import datetime
import math
from distutils.version import LooseVersion
from argparse import Namespace
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter
from torch.optim.lr_scheduler import LambdaLR
from torchvision import datasets, transforms, models
import torch.utils.data.distributed

from torchsummary import summary
import cifar_resnet as resnet
# import horovod.torch as hvd
from tqdm import tqdm
from utils import *
from qhoptim.pyt import QHAdam
from os.path import dirname, abspath
import sys
root_path = dirname(dirname(abspath(__file__)))          #"/home/cys/transformer/DeepFormer/"      "G:/DeepFormer"
# sys.path.insert(1, os.path.join(sys.path[0], '..'))
sys.path.append(root_path)

import kfac     #export PYTHONPATH=$PYTHONPATH:/home/cys/net-help/kfac_distribute/
from models import config
from models.VoT import *
from vit_pytorch import ViT
from torchvision.models import resnet50
from vit_pytorch.distill import DistillableViT, DistillWrapper
from DeepGraph import *
#   python pytorch_cifar10_resnet.py --kfac-update-freq=-10 --damping=0.003 --base-lr=0.1 --model=Jaggi
#   python pytorch_cifar10_resnet.py --kfac-update-freq=10 --damping=0.003 --base-lr=0.1 --batch-size=320 --model=Jaggi --gradient_clip=agc --self_attention=gabor
STEP_FIRST = LooseVersion(torch.__version__) < LooseVersion('1.1.0')
datas_name = "cifar10"
if datas_name == "cifar100":
    datas = datasets.CIFAR100
    nClass = 100
    # IMAGE_W,IMAGE_H=32,32
    IMAGE_W,IMAGE_H=64,64
else:
    datas = datasets.CIFAR10
    nClass = 10
    # IMAGE_W,IMAGE_H=32,32
    IMAGE_W,IMAGE_H=64,64
# Training settings
# parser = argparse.ArgumentParser(description='PyTorch CIFAR10/100 Example')
# parser.add_argument('--model', type=str, default='resnet32',
#                     help='ResNet model to use [20, 32, 56]')
# parser.add_argument('--batch-size', type=int, default=128, metavar='N',
#                     help='input batch size for training (default: 128)')
# parser.add_argument('--test-batch-size', type=int, default=128, metavar='N',
#                     help='input batch size for testing (default: 128)')
# parser.add_argument('--epochs', type=int, default=200, metavar='N',
#                     help='number of epochs to train (default: 200)')
# parser.add_argument('--warmup-epochs', type=int, default=5, metavar='WE',
#                     help='number of warmup epochs (default: 5)')
# parser.add_argument('--batches-per-allreduce', type=int, default=1,
#                     help='number of batches processed locally before '
#                          'executing allreduce across workers; it multiplies '
#                          'total batch size.')

# # Optimizer Parameters
# parser.add_argument('--base-lr', type=float, default=0.1, metavar='LR',
#                     help='base learning rate (default: 0.1)')
# parser.add_argument('--lr-decay', nargs='+', type=int, default=[100, 150],
#                     help='epoch intervals to decay lr')
# parser.add_argument('--momentum', type=float, default=0.9, metavar='M',
#                     help='SGD momentum (default: 0.9)')
# parser.add_argument('--weight-decay', type=float, default=5e-4, metavar='W',
#                     help='SGD weight decay (default: 5e-4)')

# # KFAC Parameters
# parser.add_argument('--kfac-update-freq', type=int, default=10,
#                     help='iters between kfac inv ops (0 for no kfac updates) (default: 10)')
# parser.add_argument('--kfac-cov-update-freq', type=int, default=1,
#                     help='iters between kfac cov ops (default: 1)')
# parser.add_argument('--kfac-update-freq-alpha', type=float, default=10,
#                     help='KFAC update freq multiplier (default: 10)')
# parser.add_argument('--kfac-update-freq-schedule', nargs='+', type=int, default=None,
#                     help='KFAC update freq schedule (default None)')
# parser.add_argument('--stat-decay', type=float, default=0.95,
#                     help='Alpha value for covariance accumulation (default: 0.95)')
# parser.add_argument('--damping', type=float, default=0.003,
#                     help='KFAC damping factor (defaultL 0.003)')
# parser.add_argument('--damping-alpha', type=float, default=0.5,
#                     help='KFAC damping decay factor (default: 0.5)')
# parser.add_argument('--damping-schedule', nargs='+', type=int, default=None,
#                     help='KFAC damping decay schedule (default None)')

# parser.add_argument('--kl-clip', type=float, default=0.001,help='KL clip (default: 0.001)')
# parser.add_argument('--gradient_clip', type=str, default="agc",help='KL clip (default: 0.001)')
# parser.add_argument('--self_attention', type=str, default="gaussian",help='KL clip (default: 0.001)')

# parser.add_argument('--diag-blocks', type=int, default=1,
#                     help='Number of blocks to approx layer factor with (default: 1)')
# parser.add_argument('--diag-warmup', type=int, default=5,
#                     help='Epoch to start diag block approximation at (default: 5)')
# parser.add_argument('--distribute-layer-factors', action='store_true', default=False,
#                     help='Compute A and G for a single layer on different workers')

# # Other Parameters
# parser.add_argument('--log-dir', default=f'./logs/{datas_name}/',help='TensorBoard log directory')
# #/home/cys/Downloads/{datas_name}/
# parser.add_argument('--dir', type=str, default=f'C:/Users/cys/Downloads/{datas_name}/', metavar='D',help='directory to download dataset to')
# parser.add_argument('--no-cuda', action='store_true', default=False,
#                     help='disables CUDA training')
# parser.add_argument('--seed', type=int, default=42, metavar='S',
#                     help='random seed (default: 42)')
# parser.add_argument('--fp16-allreduce', action='store_true', default=False,
#                     help='use fp16 compression during allreduce')
# parser.add_argument('--positional_encoding', type=str, default=f'')

# args = parser.parse_args()
# args.cuda = not args.no_cuda and torch.cuda.is_available()
# args.stat_decay = 0       #nealy same
isHVD = "horovod" in sys.modules
verbose = True
device = 0
sGPU = ""
download = True
num_replicas=1
rank=0

class DeepLogger(SummaryWriter):
    def __init__(self,*args, **kwargs):
        super().__init__(*args, **kwargs)
        self.epoch = -1
        self.batch_idx = -1

    def isPlot(self):
        return False
        #return self.batch_idx==2

def clip_grad(model,eps = 1.e-3,clip=0.02,method="agc"):
    known_modules = {'Linear'}
    for module in model.modules():
        classname = module.__class__.__name__
        if classname not in known_modules:
            continue
        if classname == 'Conv2d':
            assert(False)
            grad = None
        elif classname == 'BertLayerNorm':
            grad = None
        else:
            grad = module.weight.grad.data
            W = module.weight.data

        #   adaptive_grad_clip
        assert len(grad.shape)==2
        nR,nC = grad.shape
        axis = 1 if nR>nC else 0
        g_norm = unitwise_norm(grad,axis=axis)
        W_norm = unitwise_norm(W,axis=axis)
        W_norm[W_norm<eps] = eps
        # clipped_grad = grad * (W_norm / g_norm)
        s = torch.squeeze(clip*W_norm / (g_norm+1.0e-6))
        s = torch.clamp(s, max=1)
        if s.numel()==nC:       #nC
            grad = grad*s
        else:                   #nR
            grad = torch.einsum('rc,r->rc', grad, s)
        module.weight.grad.data.copy_(grad)

        if module.bias is not None:
            pass    #grad = torch.cat([grad, module.bias.grad.data.view(-1, 1)], 1)

def train(epoch):
    log_writer.epoch = epoch
    model.train()
    train_sampler.set_epoch(epoch)
    train_loss = Metric('train_loss')
    train_accuracy = Metric('train_accuracy')

    if STEP_FIRST:
        for scheduler in lr_scheduler:
            scheduler.step()
        if use_kfac:
            kfac_param_scheduler.step(epoch)
    x_info = ""
    t0 = time.time()
    batch_size = config.batch_size
    report_frequency = (int)(len(train_loader)/100.0+0.5)
    # with tqdm(total=len(train_loader), position=0, leave=True, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}',
    #           desc='Epoch {:3d}/{:3d}'.format(epoch + 1, config.epochs),
    #           disable=not verbose) as t:      always new line in WINDOWS!!!
    for batch_idx, (data, target) in enumerate(train_loader):
        log_writer.batch_idx = batch_idx
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        optimizer.zero_grad()

        for i in range(0, len(data),batch_size ):
            data_batch = data[i:i + batch_size]
            target_batch = target[i:i + batch_size]
            if model_name == "distiller":
                loss = distiller(data_batch, target_batch)
                output = model(data_batch)
            else:
                output = model(data_batch)
                if model_name == "vit":
                    output = F.log_softmax(output, dim=1)
                    loss = F.nll_loss(output, target_batch)
                else:
                    loss = criterion(output, target_batch)
            with torch.no_grad():
                train_loss.update(loss)
                train_accuracy.update(accuracy(output, target_batch))
            loss.div_(math.ceil(float(len(data)) / batch_size))
            loss.backward()
        if isHVD:
            optimizer.synchronize()
        acc = train_accuracy.avg.item()
        if use_kfac:
            preconditioner.step(epoch=epoch,accuracy=acc)
            x_info = preconditioner.dump(epoch + 1,log_writer)
        if isHVD:
            with optimizer.skip_synchronize():
                optimizer.step()
        else:
            clip_grad(model)   #need this if LayerNormal=identity
            optimizer.step()

        lr_opt = optimizer.param_groups[0]['lr']
        # t.set_postfix_str("lr={:.6f}, loss: {:.4f}, acc: {:.2f}%, {},T={:.1f}".format(lr_opt,train_loss.avg.item(), 100*acc,x_info,time.time()-t0))
        # t.update(1)
        if batch_idx % report_frequency == 0:
            print(f"\rEpoch {epoch + 1:3d}/{config.epochs:3d}\t{batch_idx}/{len(train_loader)} \tlr={lr_opt:.3f}, loss: {train_loss.avg.item():.4f}, acc: {100*acc:.2f}%, {x_info},T={time.time()-t0:.1f}",end="")
    print(f"")

    if not STEP_FIRST:
        for scheduler in lr_scheduler:
            scheduler.step()
            # print(f'\t{epoch} scheduler.get_last_lr={scheduler.get_last_lr()[0]:.6f}')
        if use_kfac:
            kfac_param_scheduler.step(epoch)

    if log_writer:
        log_writer.add_scalar('train/loss', train_loss.avg, epoch)
        log_writer.add_scalar('train/accuracy', train_accuracy.avg, epoch)

def test(epoch):
    model.eval()
    test_loss = Metric('val_loss')
    test_accuracy = Metric('val_accuracy')

    with tqdm(total=len(test_loader), bar_format='{l_bar}{bar:10}|{postfix}',
              desc='             '.format(epoch + 1, config.epochs),
              disable=not verbose) as t:
        with torch.no_grad():
            for i, (data, target) in enumerate(test_loader):
                if args.cuda:
                    data, target = data.cuda(), target.cuda()
                output = model(data)
                if model_name == "vit":
                    output = F.log_softmax(output, dim=1)
                    test_loss.update(F.nll_loss(output, target))
                else:
                    test_loss.update(criterion(output, target))
                test_accuracy.update(accuracy(output, target))

                t.update(1)
                if i + 1 == len(test_loader):
                    t.set_postfix_str("\b\b test_loss: {:.4f}, test_acc: {:.2f}%".format(
                            test_loss.avg.item(), 100*test_accuracy.avg.item()),
                            refresh=False)

    if log_writer:
        log_writer.add_scalar('test/loss', test_loss.avg, epoch)
        log_writer.add_scalar('test/accuracy', test_accuracy.avg, epoch)

if __name__ == "__main__":
    # deep_graph_info_demo()
    parser = config.get_training_parser(datas_name,desc='PyTorch CIFAR10/100 Example')
    args = config.parse_args_and_arch(parser)
    args.cuda = not args.no_cuda and torch.cuda.is_available()

    if isHVD:      #So strange!!! this would affect by TensorBoard
        print(f"hvd={hvd.local_rank()} size={hvd.size()}")
        verbose = True if hvd.rank() == 0 else False
        device = hvd.local_rank()
        sGPU = f"gpu*{hvd.size()}" if hvd.size()>1 else ""
        download = True if hvd.local_rank() == 0 else False
        if not download: hvd.allreduce(torch.tensor(1), name="barrier")
        num_replicas=hvd.size()
        rank=hvd.rank()
        args.base_lr = args.base_lr * hvd.size()
    if args.cuda:
        torch.cuda.set_device(device)
        torch.cuda.manual_seed(args.seed)
    torch.manual_seed(args.seed)
    torch.backends.cudnn.benchmark = True
    kfac.seed_everything(42)
    sFac = f"KFAC_" if args.kfac_update_freq>0 else ""
# args.log_dir = os.path.join(args.log_dir, "__{}_{}_{}_{}".format(args.model, sFac, sGPU,datetime.datetime.now().strftime('%m-%d_%H-%M')))
    print(args)
    # os.makedirs(args.log_dir, exist_ok=True)
    # log_writer = SummaryWriter(args.log_dir) if verbose else None

    # Horovod: limit # of CPU threads to be used per worker.
    torch.set_num_threads(4)

    kwargs = {'num_workers': 4, 'pin_memory': True} if args.cuda else {}

    transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.Resize([IMAGE_W,IMAGE_H]),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
        # transforms.RandomErasing(),      #p=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False
        # random_erase( p=0.5, area_ratio_range=(0.02, 0.3), min_aspect_ratio=0.1, max_attempt=6),
        # transforms.ToTensor(),
        ])
    transform_test = transforms.Compose([
        transforms.Resize([IMAGE_W,IMAGE_H]),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))

        ])

    train_dataset = datas(root=args.dir, train=True,download=download, transform=transform_train)
    test_dataset = datas(root=args.dir, train=False,download=download, transform=transform_test)

    config = args
    args.log_dir = os.path.join(f"{root_path}/logs/", "{}/__{}_{}_{}_{}".format(datas_name,args.model, sFac, sGPU,datetime.datetime.now().strftime('%m-%d_%H-%M')))
    os.makedirs(args.log_dir, exist_ok=True)
    # log_writer = SummaryWriter(args.log_dir) if verbose else None
    log_writer = DeepLogger(args.log_dir) if verbose else None
    model_name = args.model.lower()

    if model_name == "resnet20":
        model = resnet.resnet20()
    elif model_name == "resnet32":
        model = resnet.resnet32()
    elif model_name == "resnet44":
        model = resnet.resnet44()
    elif model_name == "resnet56":
        model = resnet.resnet56()
    elif model_name == "resnet110":
        model = resnet.resnet110()
    elif model_name == "vit":
        #hidden=256,very BAD!!!
        #lr=0.001 nearly same
        model = ViT(image_size = 32,patch_size = 4,num_classes = nClass,dim = 21,depth = 6,heads = 3,ff_hidden = 128,dropout = 0,emb_dropout = 0.1)
        # model = ImageTransformer(image_size=32, patch_size=4, num_classes=nClass, channels=3,dim=64, depth=6, heads=8, mlp_dim=128)          #
        # model = ViT(image_size = 256,patch_size = 32,num_classes = 1000,dim = 1024,depth = 6,eads = 16,mlp_dim = 2048,dropout = 0.1,emb_dropout = 0.1)
        #24 overfit
    elif model_name == "distiller":
        teacher = resnet50(pretrained = True)
        teacher.cuda()
        model = DistillableViT(image_size = 32,patch_size = 4,num_classes = nClass,dim = 64,depth = 6,heads = 8,mlp_dim = 128,dropout = 0.1,emb_dropout = 0.1)
        distiller = DistillWrapper(student = model,teacher = teacher,temperature = 3,alpha = 0.5)
    elif model_name == "lamlay":
        args.batch_size = 128;          args.weight_decay=0.0001
        # model = lambda_resnet26()
        model = LambdaResNet18()
        args.log_dir=f"./logs/lamlay/"
    elif model_name == "jaggi":
        VoT_config['use_attention'] = config.self_attention
        VoT_config['gradient_clip'] = config.gradient_clip
        VoT_config['INPUT_W'] = (int)(IMAGE_W / VoT_config['pooling_concatenate_size'])
        VoT_config['INPUT_H'] = (int)(IMAGE_H / VoT_config['pooling_concatenate_size'])

        VoT_config['logger'] = log_writer
        VoT_config['positional_encoding'] = config.positional_encoding
        model = VoT(VoT_config, num_classes=nClass)
        config = Namespace(**VoT_config)
        # args.log_dir=f"/home/cys/net-help/kfac_distribute/logs/Jaggi/"

    # config.log_writer = log_writer
    # module_stat(model, (3, 64, 64))
    # g = plot_graph(model, torch.zeros([1, 3, 64, 64]),path="./2.pdf")        #one more pass, would affect the training curve
    print(model)

    batch_size = config.batch_size
    if download and isHVD: hvd.allreduce(torch.tensor(1), name="barrier")
    # Horovod: use DistributedSampler to partition the training data.
    train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas=num_replicas, rank=rank)
    train_loader = torch.utils.data.DataLoader(train_dataset,
            batch_size=batch_size * args.batches_per_allreduce,
            sampler=train_sampler, **kwargs)

    # Horovod: use DistributedSampler to partition the test data.
    test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas=num_replicas, rank=rank)
    test_loader = torch.utils.data.DataLoader(test_dataset,
            batch_size=batch_size, sampler=test_sampler, **kwargs)

    if args.cuda:
        model.cuda()
    # if verbose:    summary(model, (3, 32, 32))

    criterion = nn.CrossEntropyLoss()
    use_kfac = True if args.kfac_update_freq > 0 else False

    # if use_kfac:    args.base_lr = 0.003     #0.003 for vit
    lr_scheduler=[]
    optimizer = optim.SGD(model.parameters(), lr=args.base_lr, momentum=args.momentum,weight_decay=args.weight_decay)
    if model_name == "vit" or model_name == "distiller":
        optimizer = optim.Adam(model.parameters(), lr=0.003)        #QHAdam is nearly same as adam, much better than SGD
        # optimizer = optim.SGD(model.parameters(), lr=0.03, momentum=args.momentum,weight_decay=args.weight_decay)
    if model_name == "jaggi":
        optimizer, lrs = Jaggi_get_optimizer(train_loader,model.named_parameters(),VoT_config)
        # lr_scheduler.append(lrs)

    if use_kfac:
        kfac_core = kfac.CG_KFAC if False else kfac.KFAC
        # preconditioner = kfac.KFAC(model, lr=args.base_lr, factor_decay=args.stat_decay,
        #                            damping=args.damping, kl_clip=args.kl_clip,
        #                            fac_update_freq=args.kfac_cov_update_freq,
        #                            kfac_update_freq=args.kfac_update_freq,
        #                            diag_blocks=args.diag_blocks,
        #                            diag_warmup=args.diag_warmup,
        #                            distribute_layer_factors=args.distribute_layer_factors)
        preconditioner = kfac_core(model, lr=args.base_lr, factor_decay=args.stat_decay,
                                damping=args.damping, kl_clip=args.kl_clip,
                                gradient_clip = config.gradient_clip,
                                fac_update_freq=args.kfac_cov_update_freq,
                                kfac_update_freq=args.kfac_update_freq,
                                diag_blocks=args.diag_blocks,
                                diag_warmup=args.diag_warmup,
                                distribute_layer_factors=args.distribute_layer_factors)
        kfac_param_scheduler = kfac.KFACParamScheduler(preconditioner,
                damping_alpha=args.damping_alpha,
                damping_schedule=args.damping_schedule,
                update_freq_alpha=args.kfac_update_freq_alpha,
                update_freq_schedule=args.kfac_update_freq_schedule)
    else:
        preconditioner = None

    print(f"======== optimizer={optimizer}\n\n======== MODEL={model.name_()}\n======== preconditioner={preconditioner}")
    # KFAC guarentees grads are equal across ranks before opt.step() is called
    # so if we do not use kfac we need to wrap the optimizer with horovodcon
    if isHVD:
        compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
        optimizer = hvd.DistributedOptimizer(optimizer,
                                            named_parameters=model.named_parameters(),
                                            compression=compression,
                                            op=hvd.Average,
                                            backward_passes_per_step=args.batches_per_allreduce)
        hvd.broadcast_optimizer_state(optimizer, root_rank=0)
        hvd.broadcast_parameters(model.state_dict(), root_rank=0)

    if len(lr_scheduler)==0:
        #...5...[100, 150]...
        lrs = create_lr_schedule(num_replicas, args.warmup_epochs, args.lr_decay)
        lr_scheduler = [LambdaLR(optimizer, lrs)]
    if use_kfac:
        lr_scheduler.append(LambdaLR(preconditioner, lrs))
    for ls in lr_scheduler:
        print(f"======== lr_scheduler={ls.state_dict()}")
    start = time.time()

    for epoch in range(config.epochs):
        train(epoch)
        test(epoch)

    if verbose:
        print("\nTraining time:", str(datetime.timedelta(seconds=time.time() - start)))

    print(f"======== args={args}\n")
    print(f"======== config={config}\n")
    print(f"======== B={optimizer}\n======== preconditioner={preconditioner}")
    print(f"======== optimizer={optimizer}\n======== preconditioner={preconditioner}")


import random
import argparse
from collections import defaultdict

def main(args):
    random.seed(0)

    dir = args.aurora4_list_dir

    for i in range(0, 14):
        index = f"{i+1:02}"

        spkid2utt = defaultdict(lambda: [])
        with open(f"{dir}/devtest{index}_1206_16k.list") as src_file:
            for line in src_file:
                wav_path = line.strip()
                wav_name = wav_path.split("/")[-1]
                spkid = wav_name[:3]
                spkid2utt[spkid] += [wav_name]

        prefix = "/".join(line.split("/")[:2])

        with open(f"{dir}/devtest{index}_0330a_16k.list", "w") as dst_file:
            for spkid in sorted(spkid2utt):
                sample_utt_list = random.sample(spkid2utt[spkid], 33)
                for utt in sorted(sample_utt_list):
                    dst_file.write(f"{prefix}/{spkid}_16k/{utt}\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("aurora4_list_dir", default="/mnt/HDD/dataset/Aurora4/4A/lists", nargs="?",
                        help="where is Aurora4/4a/lists")
    args = parser.parse_args()

    main(args)


from django.shortcuts import render
from .forms import userdataForm
from .models import userinfo

# Create your views here.

def index(request):
    if request.method=='POST':
        newuser=userdataForm(request.POST)
        if newuser.is_valid(): #true
            newuser.save()
            print("Your data has been saved!")
        else:
            print(newuser.errors)
    return render(request,'index.html')

def showdata(request):
    data=userinfo.objects.all()
    return render(request,'showdata.html',{'data':data})


import numpy as np
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

# --------------------------------------------
# Updates a graph step by step during training
# --------------------------------------------

# [To be done]

class Graph:

	def __init__(self, folder, name):

		self.memory = ResultMemory()
		self.folder = folder
		self.name = name
		self.filename_graph = self.folder + "/" + self.name + "_graph_"
		self.filename_metric = self.folder + "/" + self.name + "_metrics.log"
		self.results_name = None
		self.number_of_column = 0
		self.title = ""

	def set_title(self, title):
		self.title = title

	def set_names(self, new_names):
		self.results_name = new_names
		self.number_of_column = len(new_names)
		self.memory.save_new(self.filename_metric,self.results_name)

	def update(self, new_result, save = False):
		self.memory.add_result(new_result)
		if save:
			for i in np.arange(self.number_of_column):
				self.save_graph(i)
		self.save_metric(new_result)

	def save_graph(self,column_number):

		data = self.memory.get_column(column_number)
		number_of_steps = np.arange(len(data))

		plt.figure(figsize=(12, 14))

		#Axis transforms
		ax = plt.subplot(111)
		ax.spines["top"].set_visible(False)
		ax.spines["bottom"].set_visible(False)
		ax.spines["right"].set_visible(False)
		ax.spines["left"].set_visible(False)
		ax.get_xaxis().tick_bottom()
		ax.get_yaxis().tick_left()

		plt.suptitle(self.title + self.results_name[column_number], fontsize=17, ha="center")

		plt.plot(number_of_steps, data, lw=2.5)

		plt.savefig((self.filename_graph +self.results_name[column_number] + ".png"), bbox_inches="tight")

	def save_metric(self, new_results):
		self.memory.save_new(self.filename_metric, new_results)

class ResultMemory():

	def __init__(self):

		self.result_holder = []	# Holds a list of lists of values

	def add_result(self,new_result):

		if type(new_result) == None:
			print("WARGNING : Result to be saved are empty - No result saved at this step (In PyDS.Evaluation.Graphs.ResultMemory)")
			return

		if type(new_result) == type(np.ndarray([0,0])):
			new_result = new_result.tolist()

		if type(new_result) == type(self.result_holder):
			if len(self.result_holder) == 0:
				self.result_holder.append(new_result)
			else :
				if len(new_result) != len(self.result_holder[0]):
					print("Warning : Length of result is changed, results might be inconsistent or missing (In PyDS.Evaluation.Graphs.ResultMemory)")
					self.result_holder.append(new_result)
				else:
					self.result_holder.append(new_result)

		else :
			print("Type of new result is not a list, it is : ", type(new_result))

	def get_results(self):
		return self.result_holder

	def get_column(self, column_number):
		column_result = []

		for iterator in self.result_holder:
			column_result.append(iterator[column_number])

		return column_result

	def save_new(self, filename, new_result):
		log = open(filename, "a")

		for result in new_result:
			log.write(str(result))
			log.write("  ;  ")
		log.write("\n")
		log.close()

def saveParameters(savepath, args):

	log = open(savepath, 'w')
	log.write("Fisheye camera file: " + args.fisheye+"\n")
	log.write("Thermal camera file: " + args.thermal+"\n")
	log.write("Device: " + args.device+"\n")
	log.write("Detection task"+"\n")
	log.write("Yolo configuration: "+ args.yoloconfig + "\n")
	if args.weights is not None:
		log.write("Using weights: " + args.weights+"\n")
	else:
		log.write("From scratch"+"\n")
	log.write("Learning rate: " + str(args.learningrate)+"\n")
	log.write("Training set size: " + str(args.trainsetsize)+"\n")
	log.write("Student subsample: " + str(args.studentsubsample)+"\n")
	log.write("Teacher subsample: " + str(args.teachersubsample)+"\n")
	log.close()