import hid
# import hid
# pip install hidapi
# https://github.com/trezor/cython-hidapi
import time
import os

PACKET_SIZE = 65
DIR_OUTPUT = 0
DIR_INPUT = 1

class PyMCP2221A:
    def __init__(self, VID=0x04D8, PID=0x00DD, devnum=0):
        self.mcp2221a = hid.device()
        self.mcp2221a.open_path(hid.enumerate(VID, PID)[devnum]["path"])
        self.CLKDUTY_0 = 0x00
        self.CLKDUTY_25 = 0x08
        self.CLKDUTY_50 = 0x10
        self.CLKDUTY_75 = 0x18
        # self.CLKDIV_1 = 0x00    # 48MHz  Dont work.
        self.CLKDIV_2 = 0x01  # 24MHz
        self.CLKDIV_4 = 0x02  # 12MHz
        self.CLKDIV_8 = 0x03  # 6MHz
        self.CLKDIV_16 = 0x04  # 3MHz
        self.CLKDIV_32 = 0x05  # 1.5MHz
        self.CLKDIV_64 = 0x06  # 750KHz
        self.CLKDIV_128 = 0x07  # 375KHz

    def compile_packet(self, buf):
        """
        :param list buf:
        """
        assert len(buf) <= PACKET_SIZE

        buf = buf + [0 for i in range(PACKET_SIZE - len(buf))]
        return buf

    #######################################################################
    # HID DeviceDriver Info
    #######################################################################
    def DeviceDriverInfo(self):
        print("Manufacturer: %s" % self.mcp2221a.get_manufacturer_string())
        print("Product: %s" % self.mcp2221a.get_product_string())
        print("Serial No: %s" % self.mcp2221a.get_serial_number_string())

    #######################################################################
    # Command Structure
    #######################################################################
    def Command_Structure(self, I2C_Cancel_Bit, I2C_Speed_SetUp_Bit, I2C_Speed_SetVal_Byte):
        I2C_Cancel_Bit = 0
        I2C_Speed_SetUp_Bit = 0
        I2C_Speed_SetVal_Byte = 0
        buf = self.compile_packet([0x00, 0x10, 0x00,
                                   I2C_Cancel_Bit << 4,
                                   I2C_Speed_SetUp_Bit << 5,
                                   I2C_Speed_SetVal_Byte])

        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

        print(chr(buf[46]))
        print(chr(buf[47]))
        print(chr(buf[48]))
        print(chr(buf[49]))

    #######################################################################
    # Read Flash Data
    #######################################################################

    def Read_Flash_Data(self, Read_Deta_Setting_Byte):
        Read_Deta_Setting_Byte = 0x00
        # Read_Chip_Settings             = 0x00
        # Read_GP_Settings               = 0x01
        # Read_USB_Manufacturer_Settings = 0x02
        # Read_USB_Product_Settings      = 0x03
        # Read_USB_SerialNum_Settings    = 0x04
        # Read_Chip_Factory_Settings     = 0x05

        buf = self.compile_packet([0x00, 0xB0, Read_Deta_Setting_Byte])
        # print ("Write")
        # print (buf)
        self.mcp2221a.write(buf)

        buf = self.mcp2221a.read(PACKET_SIZE)
        # print ("Read")
        # print (buf)

    #######################################################################
    # Write Flash Data
    #######################################################################

    def Write_Flash_Data(self, data):
        pass
        # Write_Deta_Setting_Byte = 0x00
        # Write_Chip_Settings             = 0x00
        # Write_GP_Settings               = 0x01
        # Write_USB_Manufacturer_Settings = 0x02
        # Write_USB_Product_Settings      = 0x03
        # Write_USB_SerialNum_Settings    = 0x04
        # buf = [0x00,0xB1,Write_Deta_Setting_Byte]
        # buf = buf + [0 for i in range(PACKET_SIZE-len(buf))]
        # !!!! Be careful when making changes !!!!
        # buf[6+1] =  0xD8    # VID (Lower)
        # buf[7+1] =  0x04    # VID (Higher)
        # buf[8+1] =  0xDD    # PID (Lower)
        # buf[9+1] =  0x00    # PID (Higher)

        # print ("Write")
        # print (buf)
        # h.write(buf)
        # buf = h.read(PACKET_SIZE)
        # print ("Read")
        # print (buf)

    #######################################################################
    # GPIO Init
    #######################################################################
    def GPIO_Init(self):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = rbuf[6]  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = 0x00  # ADC Voltage Reference
        # buf[6+1] = 0x00     #   Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0x80  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80

        self.GPIO_0_BIT = (rbuf[22 + 1] >> 4) & 0x01  # 1:Hi 0:LOW
        self.GPIO_0_DIR = (rbuf[22 + 1] >> 3) & 0x01  # 0:OutPut 1:Input
        self.GPIO_0_MODE = rbuf[22 + 1] & 0x07  # GPIO MODE = 0x00
        self.GPIO_1_BIT = (rbuf[23 + 1] >> 4) & 0x01  # 1:Hi 0:LOW
        self.GPIO_1_DIR = (rbuf[23 + 1] >> 3) & 0x01  # 0:OutPut 1:Input
        self.GPIO_1_MODE = rbuf[23 + 1] & 0x07  # GPIO MODE = 0x00
        self.GPIO_2_BIT = (rbuf[24 + 1] >> 4) & 0x01  # 1:Hi 0:LOW
        self.GPIO_2_DIR = (rbuf[24 + 1] >> 3) & 0x01  # 0:OutPut 1:Input
        self.GPIO_2_MODE = rbuf[24 + 1] & 0x07  # GPIO MODE = 0x00
        self.GPIO_3_BIT = (rbuf[25 + 1] >> 4) & 0x01  # 1:Hi 0:LOW
        self.GPIO_3_DIR = (rbuf[25 + 1] >> 3) & 0x01  # 0:OutPut 1:Input
        self.GPIO_3_MODE = rbuf[25 + 1] & 0x07  # GPIO MODE = 0x00

        # for(i in range(64)):
        #    buf[i] = rbuf[i] | buf[i]
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # GPIO Set Direction and Set Value commands
    #######################################################################
    def GPIO_SetDirection(self, pin, direction):

        buf = self.compile_packet([0x00, 0x50])
        offset = (pin + 1) * 4
        buf[offset + 1] = 0x01  # set pin direction
        buf[offset + 1 + 1] = direction  # to this
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)
        if rbuf[1] != 0x00:
            raise RuntimeError("GPIO_SetDirection Failed")

    def GPIO_SetValue(self, pin, value):

        buf = self.compile_packet([0x00, 0x50])
        offset = ((pin + 1) * 4) - 1
        buf[offset - 1 + 1] = 0x01  # set pin value
        buf[offset + 1] = value  # to this
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)
        if rbuf[1] != 0x00:
            raise RuntimeError("GPIO_SetValue Failed")

    #######################################################################
    # Read GPIO Data command
    #######################################################################
    def GPIO_Read(self):

        buf = self.compile_packet([0x00, 0x51])
        self.mcp2221a.write(buf)

        buf = self.mcp2221a.read(PACKET_SIZE)
        self.GPIO_0_INPUT = buf[2]
        self.GPIO_0_DIR = buf[3]
        self.GPIO_1_INPUT = buf[4]
        self.GPIO_1_DIR = buf[5]
        self.GPIO_2_INPUT = buf[6]
        self.GPIO_2_DIR = buf[7]
        self.GPIO_3_INPUT = buf[8]
        self.GPIO_3_DIR = buf[9]
        return buf

    # Return the GPIO value as an integer instead of tuple
    def GPIO_GetValue(self, pin):
        rbuf = self.GPIO_Read()
        offset = (pin + 1) * 2
        if rbuf[offset] == 0xEE:
            raise RuntimeError("GPIO_GetValue Failed, pin is not set for GPIO operation")
        return rbuf[offset]

    #######################################################################
    # GPIO Outpu/Input Data
    #######################################################################
    def GPIO_0_Output(self, bit):
        self.GPIO_0_BIT = bit  # 1:Hi 0:LOW
        self.GPIO_0_OutputMode()
        self.GPIO_SetValue(pin=0, value=self.GPIO_0_BIT)

    def GPIO_0_InputMode(self):
        self.GPIO_0_DIR = DIR_INPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=0, direction=self.GPIO_0_DIR)

    def GPIO_0_OutputMode(self):
        self.GPIO_0_DIR = DIR_OUTPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=0, direction=self.GPIO_0_DIR)

    def GPIO_0_Input(self):
        self.GPIO_Read()
        return self.GPIO_0_INPUT, self.GPIO_0_DIR

    def GPIO_1_Output(self, bit):
        self.GPIO_1_BIT = bit  # 1:Hi 0:LOW
        self.GPIO_1_OutputMode()
        self.GPIO_SetValue(pin=1, value=self.GPIO_1_BIT)

    def GPIO_1_InputMode(self):
        self.GPIO_1_DIR = DIR_INPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=1, direction=self.GPIO_1_DIR)

    def GPIO_1_OutputMode(self):
        self.GPIO_1_DIR = DIR_OUTPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=1, direction=self.GPIO_1_DIR)

    def GPIO_1_Input(self):
        self.GPIO_Read()
        return self.GPIO_1_INPUT, self.GPIO_1_DIR

    def GPIO_2_Output(self, bit):
        self.GPIO_2_BIT = bit  # 1:Hi 0:LOW
        self.GPIO_2_OutputMode()
        self.GPIO_SetValue(pin=2, value=self.GPIO_2_BIT)

    def GPIO_2_InputMode(self):
        self.GPIO_2_DIR = DIR_INPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=2, direction=self.GPIO_2_DIR)

    def GPIO_2_OutputMode(self):
        self.GPIO_2_DIR = DIR_OUTPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=2, direction=self.GPIO_2_DIR)

    def GPIO_2_Input(self):
        self.GPIO_Read()
        return self.GPIO_2_INPUT, self.GPIO_2_DIR

    def GPIO_3_Output(self, bit):
        self.GPIO_3_BIT = bit  # 1:Hi 0:LOW
        self.GPIO_3_OutputMode()
        self.GPIO_SetValue(pin=3, value=self.GPIO_3_BIT)

    def GPIO_3_InputMode(self):
        self.GPIO_3_DIR = DIR_INPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=3, direction=self.GPIO_3_DIR)

    def GPIO_3_OutputMode(self):
        self.GPIO_3_DIR = DIR_OUTPUT  # 0:OutPut 1:Input
        self.GPIO_SetDirection(pin=3, direction=self.GPIO_3_DIR)

    def GPIO_3_Input(self):
        self.GPIO_Read()
        return self.GPIO_3_INPUT, self.GPIO_3_DIR

    #######################################################################
    # Clock Out Value & Duty
    #######################################################################
    def ClockOut(self, duty, value):
        """
        :param int duty:    Bit 4-3: Duty cycle
                            00 | 0% duty cycle
                            01 | 25% duty cycle
                            10 | 50% duty cycle
                            11 | 75% duty cycle
        :param int value:   Bit 2-0: Clock divider value
        """
        buf = self.compile_packet([0x00, 0x61])  # Get SRAM Settings

        self.mcp2221a.write(buf)
        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])  # Set SRAM Settings
        buf[2 + 1] = 0x80 | duty | (0x07 & value)  # Clock Output Divider value
        buf[3 + 1] = rbuf[6]  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0x80  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = 0x01  # GP1 settings - 001 Dedicated function operation (Clock Output)
        buf[10 + 1] = rbuf[24]  # GP2 settings
        buf[11 + 1] = rbuf[25]  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # ADC 1
    #######################################################################
    def ADC_1_Init(self):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = rbuf[6]  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = 0x00  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0xFF  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = 0x02  # GP1 settings
        buf[10 + 1] = rbuf[24]  # GP2 settings
        buf[11 + 1] = rbuf[25]  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # ADC 2
    #######################################################################

    def ADC_2_Init(self):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = rbuf[6]  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0x80  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = rbuf[23]  # GP1 settings
        buf[10 + 1] = 0x02  # GP2 settings
        buf[11 + 1] = rbuf[25]  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # ADC 3
    #######################################################################

    def ADC_3_Init(self):
        buf = self.compile_packet([0x00, 0x61])

        self.mcp2221a.write(buf)
        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = rbuf[6]  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0x80  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = rbuf[23]  # GP1 settings
        buf[10 + 1] = rbuf[24]  # GP2 settings
        buf[11 + 1] = 0x02  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # ADC Deta Get
    #######################################################################

    def ADC_DataRead(self):
        buf = self.compile_packet([0x00, 0x10])
        self.mcp2221a.write(buf)

        buf = self.mcp2221a.read(PACKET_SIZE)
        # for i in range(len(buf)):
        #    print ("[%d]: 0x{:02x}".format(buf[i]) % (i))
        self.ADC_1_data = buf[50] | (buf[51] << 8)  # ADC Data (16-bit) values
        self.ADC_2_data = buf[52] | (buf[53] << 8)  # ADC Data (16-bit) values
        self.ADC_3_data = buf[54] | (buf[55] << 8)  # ADC Data (16-bit) values

    #######################################################################
    # DAC 1
    #######################################################################
    def DAC_1_Init(self):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = 0x00  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0xFF  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = rbuf[23]  # GP1 settings
        buf[10 + 1] = 0x03  # GP2 settings
        buf[11 + 1] = rbuf[25]  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # DAC 2
    #######################################################################

    def DAC_2_Init(self):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = 0x00  # DAC Voltage Reference
        buf[4 + 1] = 0x00  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0xFF  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = rbuf[23]  # GP1 settings
        buf[10 + 1] = rbuf[24]  # GP2 settings
        buf[11 + 1] = 0x03  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # DAC Output
    #######################################################################

    def DAC_Datawrite(self, value):
        buf = self.compile_packet([0x00, 0x61])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)

        buf = self.compile_packet([0x00, 0x60])
        buf[2 + 1] = rbuf[5]  # Clock Output Divider value
        buf[3 + 1] = 0x00  # DAC Voltage Reference
        buf[4 + 1] = 0x80 | (0x1F & value)  # Set DAC output value
        buf[5 + 1] = rbuf[7]  # ADC Voltage Reference
        buf[6 + 1] = 0x00  # Setup the interrupt detection mechanism and clear the detection flag
        buf[7 + 1] = 0xFF  # Alter GPIO configuration: alters the current GP designation
        #   datasheet says this should be 1, but should actually be 0x80
        buf[8 + 1] = rbuf[22]  # GP0 settings
        buf[9 + 1] = rbuf[23]  # GP1 settings
        buf[10 + 1] = rbuf[24]  # GP2 settings
        buf[11 + 1] = rbuf[25]  # GP3 settings
        self.mcp2221a.write(buf)
        buf = self.mcp2221a.read(PACKET_SIZE)

    #######################################################################
    # I2C Init
    #######################################################################
    def I2C_Init(self, speed=100000):  # speed = 100000
        self.MCP2221_I2C_SLEEP = float(os.environ.get("MCP2221_I2C_SLEEP", 0))

        buf = self.compile_packet([0x00, 0x10])
        buf[2 + 1] = 0x00  # Cancel current I2C/SMBus transfer (sub-command)
        buf[3 + 1] = 0x20  # Set I2C/SMBus communication speed (sub-command)
        # The I2C/SMBus system clock divider that will be used to establish the communication speed
        buf[4 + 1] = int((12000000 / speed) - 3)
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)
        # print("Init")
        if (rbuf[22] == 0):
            raise RuntimeError("SCL is low.")
        if (rbuf[23] == 0):
            raise RuntimeError("SDA is low.")

        # time.sleep(0.001)

    #######################################################################
    # I2C State Check
    #######################################################################
    def I2C_State_Check(self):
        buf = self.compile_packet([0x00, 0x10])
        self.mcp2221a.write(buf)

        rbuf = self.mcp2221a.read(PACKET_SIZE)
        return rbuf[8]

    #######################################################################
    # I2C Cancel
    #######################################################################

    def I2C_Cancel(self):
        buf = self.compile_packet([0x00, 0x10])
        buf[2 + 1] = 0x10  # Cancel current I2C/SMBus transfer (sub-command)
        self.mcp2221a.write(buf)

        self.mcp2221a.read(PACKET_SIZE)
        # time.sleep(0.1)

    #######################################################################
    # I2C Write
    #######################################################################
    def I2C_Write(self, addrs, data):
        """ Writes a block of data with Start and Stop c condition on bus
        :param int addrs: 7-bit I2C slave address
        :param list data: list of int

        Referring to MCP2221A Datasheet(Rev.B 2017), section 3.1.5
        """
        buf = self.compile_packet([0x00, 0x90])
        self._i2c_write(addrs, data, buf)

    def I2C_Write_Repeated(self, addrs, data):
        """ Writes a block of data with Repeated Start and Stop conditions on bus
        :param int addrs: 7-bit I2C slave address
        :param list data: list of int

        Referring to MCP2221A Datasheet(Rev.B 2017), section 3.1.6
        """
        buf = self.compile_packet([0x00, 0x92])
        self._i2c_write(addrs, data, buf)

    def I2C_Write_No_Stop(self, addrs, data):
        """ Writes a block of data with Start condition on bus
        :param int addrs: 7-bit I2C slave address
        :param list data: list of int

        Referring to MCP2221A Datasheet(Rev.B 2017), section 3.1.7
        """
        buf = self.compile_packet([0x00, 0x94])
        self._i2c_write(addrs, data, buf)

    def _i2c_write(self, addrs, data, buf):

        buf[1 + 1] = (len(data) & 0x00FF)  # Cancel current I2C/SMBus transfer (sub-command)
        buf[2 + 1] = (len(data) & 0xFF00) >> 8  # Set I2C/SMBus communication speed (sub-command)
        # The I2C/SMBus system clock divider that will be used to establish the communication speed
        buf[3 + 1] = 0xFF & (addrs << 1)
        for i in range(len(data)):
            # print ("{:d}: 0x{:02x}".format(i,data[i]))
            buf[4 + 1 + i] = data[
                i]  # The I2C/SMBus system clock divider that will be used to establish the communication speed
        self.mcp2221a.write(buf)
        rbuf = self.mcp2221a.read(PACKET_SIZE)
        time.sleep(0.008)

    #######################################################################
    # I2C Read
    #######################################################################
    def I2C_Read(self, addrs, size):
        """ Reads a block of data with Start and Stop conditions on bus
        :param int addrs: 7-bit I2C slave address
        :param int size: size of read out in bytes

        Referring to MCP2221A Datasheet(Rev.B 2017), section 3.1.8
        """
        buf = self.compile_packet([0x00, 0x91])
        return self._i2c_read(addrs, size, buf)

    def I2C_Read_Repeated(self, addrs, size):
        """ Reads a block of data with Repeated Start and Stop conditions on bus
        :param int addrs: 7-bit I2C slave address
        :param int size: size of read out in bytes

        Referring to MCP2221A Datasheet(Rev.B 2017), section 3.1.9
        """
        buf = self.compile_packet([0x00, 0x93])
        return self._i2c_read(addrs, size, buf)

    def _i2c_read(self, addrs, size, buf):

        buf[1 + 1] = (size & 0x00FF)  # Read LEN
        buf[2 + 1] = (size & 0xFF00) >> 8  # Read LEN
        buf[3 + 1] = 0xFF & (addrs << 1)  # addrs
        self.mcp2221a.write(buf)
        rbuf = self.mcp2221a.read(PACKET_SIZE)
        if (rbuf[1] != 0x00):
            # print("[0x91:0x{:02x},0x{:02x},0x{:02x}]".format(rbuf[1],rbuf[2],rbuf[3]))
            self.I2C_Cancel()
            self.I2C_Init()
            raise RuntimeError("I2C Read Data Failed: Code " + rbuf[1])
        time.sleep(self.MCP2221_I2C_SLEEP)

        buf = self.compile_packet([0x00, 0x40])
        buf[1 + 1] = 0x00
        buf[2 + 1] = 0x00
        buf[3 + 1] = 0x00
        self.mcp2221a.write(buf)
        rbuf = self.mcp2221a.read(PACKET_SIZE)
        if (rbuf[1] != 0x00):
            # print("[0x40:0x{:02x},0x{:02x},0x{:02x}]".format(rbuf[1],rbuf[2],rbuf[3]))
            self.I2C_Cancel()
            self.I2C_Init()
            print("You can try increasing environment variable MCP2221_I2C_SLEEP")
            raise RuntimeError("I2C Read Data - Get I2C Data Failed: Code " + rbuf[1])
        if (rbuf[2] == 0x00 and rbuf[3] == 0x00):
            self.I2C_Cancel()
            self.I2C_Init()
            return rbuf[4]
        if (rbuf[2] == 0x55 and rbuf[3] == size):
            rdata = [0] * size
            for i in range(size):
                rdata[i] = rbuf[4 + i]
            return rdata

    #######################################################################
    # reset
    #######################################################################
    def Reset(self):
        print("Reset")
        buf = self.compile_packet([0x00, 0x70, 0xAB, 0xCD, 0xEF])

        self.mcp2221a.write(buf)
        time.sleep(1)


import time
import asyncio
import aiohttp
import requests
import urllib3
from datetime import datetime, timedelta

from bs4 import BeautifulSoup
import pandas as pd

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

async def search_daum_news(key_row):
    keyword = key_row[0]
    x_word = key_row[1]
    sTxt = keyword + ' ' + x_word

    columns = ['제목', 'URL', '언론사', 'KeyWord', '포털명']
    df = pd.DataFrame(columns=columns)

    # 다음 검색시 헤더가 삽입되면 로봇으로 간주함
    async with aiohttp.ClientSession() as session:
        for page in range(1, 50):
            now_ymdhms = datetime.now().strftime('%Y%m%d%H%M%S')  # Today
            day_ago_ymdhms = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d%H%M%S')  # D-1 날짜

            url = f"https://search.daum.net/search?w=news&cluster=n" \
                  f"&q={keyword}" \
                  f"&sort=recency&DA=STC&period=d" \
                  f"&sd={day_ago_ymdhms}" \
                  f"&ed={now_ymdhms}" \
                  f"&p={page}"
            async with session.get(url) as resp:
                html = await resp.text()
                soup = BeautifulSoup(html, 'html.parser')
                il_Array = soup.select("ul.list_news li")

                for i in il_Array:
                    try:
                        try:
                            title = i.select_one("div.wrap_cont a").text
                        except:
                            title = None
                        try:
                            link = i.select_one("div.wrap_cont a").attrs["href"]
                        except:
                            link = None
                        try:
                            site = i.select_one("div > span.cont_info > a").text
                        except:
                            site = None
                        rNews = [title, link, site, keyword, "Daum"]
                        df = df.append(pd.Series(rNews, index=['제목', 'URL', '언론사', 'KeyWord', '포털명']), ignore_index=True)
                    except Exception as ex:
                        print(ex)

                # print(f" DAUM {keyword} {page} 검색중입니다...")

                isNext = soup.select_one("div.result_message.mg_cont.hide")
                if not isNext:
                    break

    print(f">> DAUM {keyword} 검색이 끝났습니다.")

    return df


"""added tsvector column and index for full text search

Revision ID: 81bc3c4ff4a4
Revises: e2d73cd9f007
Create Date: 2020-10-25 10:33:39.259946

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '81bc3c4ff4a4'
down_revision = 'e2d73cd9f007'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('show', sa.Column('tsv_searchable_text', postgresql.TSVECTOR(), nullable=True))
    op.create_index('tsv_idx', 'show', ['tsv_searchable_text'], unique=False, postgresql_using='gin')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('tsv_idx', table_name='show')
    op.drop_column('show', 'tsv_searchable_text')
    # ### end Alembic commands ###


from typing import Union, List
from starlette import status
import json
from fastapi.encoders import jsonable_encoder
from fastapi import APIRouter, Response, WebSocket
from fastapi_sqlalchemy import db
from src.schemas.glass import GlassType, GlassManufacture, GlassManufactureOut
from src.schemas.info import ResponseInfo
from src.models import GlassManufacture as GlassManufactureModel

router = APIRouter()

@router.get("/glass/manufacture/all", response_model=List[GlassManufactureOut])
def get_all_glass_manufactures():
    return db.session.query(GlassManufactureModel).all()

@router.post("/glass/manufacture/add", response_model=List[GlassManufactureOut])
def add_glass_manufacture(glass_manufacture: GlassManufacture):
    db_glass_manufacture = GlassManufactureModel(name=glass_manufacture.name)
    db.session.add(db_glass_manufacture)
    db.session.commit()
    return get_all_glass_manufactures()

@router.delete("/glass/manufacture/{glass_manufacture_id}",
               response_model=Union[List[GlassManufactureOut], ResponseInfo])
def delete_glass_manufacture(glass_manufacture_id: int, response: Response):
    db_glass_manufacture = get_glass_manufacture_by_id(glass_manufacture_id)
    if db_glass_manufacture is None:
        response.status_code = status.HTTP_404_NOT_FOUND
        return ResponseInfo(msg="Указанной обработки стекла не существует")
    db.session.delete(db_glass_manufacture)
    db.session.commit()
    return get_all_glass_manufactures()

@router.put("/glass/manufacture/{glass_manufacture_id}", response_model=Union[List[GlassManufactureOut], ResponseInfo])
def update_glass_manufacture(glass_manufacture_id: int, glass_manufacture: GlassManufacture, response: Response):
    db_glass_manufacture = get_glass_manufacture_by_id(glass_manufacture_id)
    if db_glass_manufacture is None:
        response.status_code = status.HTTP_404_NOT_FOUND
        return ResponseInfo(msg="Указанной обработки стекла не существует")
    db_glass_manufacture.name = glass_manufacture.name
    db.session.commit()
    return get_all_glass_manufactures()

def get_glass_manufacture_by_id(glass_manufacture_id: int) -> GlassType:
    return db.session.query(GlassManufactureModel).filter(GlassManufactureModel.id == glass_manufacture_id).first()

class ManufacturesWebsocketManager:
    def __init__(self):
        self.active_connections: List[WebSocket] = []

    async def connect(self, websocket: WebSocket):
        await websocket.accept()
        self.active_connections.append(websocket)

    def disconnect(self, websocket: WebSocket):
        try:
            self.active_connections.remove(websocket)
        except Exception as e:
            pass

    async def add_manufacture(self, data: dict, websocket: WebSocket):
        try:
            GlassManufacture.name_validator(data["name"])
        except Exception as e:
            notification = {'message_type': 'notification', 'data': {'notification_type': 'error',
                                                                     'header': 'Ошибка!',
                                                                     'message': str(e)}}
            notification_json = jsonable_encoder(notification)
            await self.send_personal_message(notification_json, websocket)
            return

        db_glass_manufacture = GlassManufactureModel(
            name=data['name']
        )
        with db():
            db.session.add(db_glass_manufacture)
            db.session.commit()
            response = {'message_type': 'get', 'data': get_all_glass_manufactures()}
            response_json = jsonable_encoder(response)
            await self.broadcast(response_json)
            notification = {'message_type': 'notification', 'data': {'notification_type': 'success',
                                                                     'header': 'Новая обработка стекла успешно добавлена!',
                                                                     'message': 'Ура!'}}
            notification_json = jsonable_encoder(notification)
            await self.send_personal_message(notification_json, websocket)

    async def update_manufacture(self, data: dict, websocket: WebSocket):
        try:
            GlassManufacture.name_validator(data["name"])
        except Exception as e:
            notification = {'message_type': 'notification', 'data': {'notification_type': 'error',
                                                                     'header': 'Ошибка!',
                                                                     'message': str(e)}}
            notification_json = jsonable_encoder(notification)
            await self.send_personal_message(notification_json, websocket)
            return

        with db():
            db_glass_type = get_glass_manufacture_by_id(int(data['id']))
            if db_glass_type is not None:
                db_glass_type.name = data['name']
                db.session.commit()
                response = {'message_type': 'get', 'data': get_all_glass_manufactures()}
                response_json = jsonable_encoder(response)
                await self.broadcast(response_json)
                notification = {'message_type': 'notification', 'data': {'notification_type': 'success',
                                                                         'header': 'Данные об обработке стекла успешно изменены!',
                                                                         'message': 'Ура!'}}
                notification_json = jsonable_encoder(notification)
                await self.send_personal_message(notification_json, websocket)
            else:
                notification = {'message_type': 'notification', 'data': {'notification_type': 'error',
                                                                         'header': 'Не удалось изменить данные об обработке стекла!',
                                                                         'message': 'Указанной обработки стекла нет в базе данных'}}
                notification_json = jsonable_encoder(notification)
                await self.send_personal_message(notification_json, websocket)

    async def delete_manufacture(self, data: dict, websocket: WebSocket):
        with db():
            db_glass_type = get_glass_manufacture_by_id(int(data['id']))
            if db_glass_type is not None:
                db.session.delete(db_glass_type)
                db.session.commit()
                response = {'message_type': 'get', 'data': get_all_glass_manufactures()}
                response_json = jsonable_encoder(response)
                await self.broadcast(response_json)
                notification = {'message_type': 'notification', 'data': {'notification_type': 'success',
                                                                         'header': 'Обработка стекла успешно удалена!',
                                                                         'message': 'Ура!'}}
                notification_json = jsonable_encoder(notification)
                await self.send_personal_message(notification_json, websocket)
            else:
                notification = {'message_type': 'notification', 'data': {'notification_type': 'error',
                                                                         'header': 'Не удалось удалить обработку стекла!',
                                                                         'message': 'Указанной обработки стекла нет в базе данных'}}
                notification_json = jsonable_encoder(notification)
                await self.send_personal_message(notification_json, websocket)

    async def get_manufactures(self, websocket: WebSocket):
        with db():
            response = {'message_type': 'get', 'data': get_all_glass_manufactures()}
            response_json = jsonable_encoder(response)
            await self.send_personal_message(response_json, websocket)

    async def send_personal_message(self, message: str, websocket: WebSocket):
        await websocket.send_json(message)

    async def broadcast(self, message: str, broadcast_except: WebSocket = None):
        for connection in self.active_connections:
            #if broadcast_except is None or broadcast_except == connection:
            await connection.send_json(message)

websocket_manager = ManufacturesWebsocketManager()

@router.websocket("/ws/glass/manufacture")
async def manufactures_websocket(websocket: WebSocket):
    print('Accepting client connection...')
    await websocket_manager.connect(websocket)
    while True:
        try:
            # Wait for any message from the client
            request_json = await websocket.receive_text()
            request = json.loads(request_json)
            print(request)
            if request['message_type'] == 'add':
                await websocket_manager.add_manufacture(request['data'], websocket)

            if request['message_type'] == 'update':
                await websocket_manager.update_manufacture(request['data'], websocket)

            if request['message_type'] == 'delete':
                await websocket_manager.delete_manufacture(request['data'], websocket)

            if request['message_type'] == 'get':
                await websocket_manager.get_manufactures(websocket)

        except Exception as e:
            websocket_manager.disconnect(websocket)
            break


# -*- coding: utf-8 -*-
from wxapp.settings import WX_URL, WX_APPID, WX_SECRET
from utils.http import requests_noauth

class WxApi(object):

    @classmethod
    def get_token(cls, wx_appid=WX_APPID, wx_secret=WX_SECRET):
        # https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=wx16be51dfaf68d05c&secret=
        # 00f6efa87e7fe2beac844e721cc21702
        url = WX_URL + "/token?" + "&".join(["grant_type=client_credential", "appid=%s" % wx_appid, "secret=%s" % wx_secret])
        r = requests_noauth("GET", url, timeout=3)
        if r.status_code // 100 != 2:
            r = r.json()
            if "access_token" in r:
                return r["access_token"], r["expires_in"]
        raise Exception("Get Token error!")


import os

# doesn't manage post with at least a million of comments

path = "C:\\Users\\Benjamin\\Desktop\\BensFolder\\DEV\\Candco\\CPP\\Projects\\YouTubeCommentsGraph\\CommentsCounter\\community\\"

os.chdir(path)

youtuber = 'JUL'#'Furious Jumper'#'Squeezie'

f = open(youtuber + '.txt', encoding = 'utf-8')
lines = f.readlines()
f.close()

pattern = 'size-default" aria-label="'

totalCommentsMinimum = 0
totalCommentsMaximum = 0

linesLen = len(lines)
for linesIndex in range(linesLen):
    line = lines[linesIndex]
    if pattern in line:
        line = line.replace(' ', '')#, ' ')
        lineParts = line.split(pattern)
        lineParts1 = lineParts[1]
        lineParts1Parts = lineParts1.split("commentaire") # used to be "commentaires" but a post of JUL has only a single comment :')
        lineParts1Parts0 = lineParts1Parts[0]
        lineParts1Parts0 = lineParts1Parts0.replace(',', '.')
        #print(lineParts1Parts0)
        factor = 1
        isThousand = lineParts1Parts0[-1] == 'k'
        if isThousand:
            factor = 1000
            lineParts1Parts0 = lineParts1Parts0[:-1]
        comments = int(float(lineParts1Parts0) * factor)
        totalCommentsMinimum += comments
        totalCommentsMaximum += comments
        if isThousand:
            totalCommentsMaximum += 1000

print(totalCommentsMinimum, totalCommentsMaximum)


import logging

import typer

from typing import Optional
from pathlib import Path
from als_recommender import ALSRecommender

logging.basicConfig(filename='../logs/app.log', level=logging.INFO, format='%(asctime)s - %(message)s',
                    datefmt='%d-%b-%y %H:%M:%S')

app = typer.Typer()
als_rec = ALSRecommender()

@app.command()
def train(dataset: Optional[Path] = Path("../data/ratings_train.dat")):
    """
    API call for model.train()
    :param dataset: path to train dataset
    :return: None
    """
    als_rec.train(dataset)

@app.command()
def evaluate(dataset: Optional[Path] = Path("../data/ratings_test.dat")):
    """
    API call to model.evaluate()
    :param dataset: path to validation dataset
    :return: None
    """
    als_rec.evaluate(dataset)

@app.command()
def predict(user_id: Optional[str]=100, M: Optional[str]=10):
    """
    API call to model.recommend()
    :param user_id: user id in System
    :param M: count of recommend films
    :return: (movies_id_list, predicted_ratings_list)
    """
    movies, preds = als_rec.recommend(user_id, m=M)
    print(f"Recommended Films: {[int(x) for x in movies]} \n with ratings: {[str(x) for x in preds]}")

    return movies, preds

@app.command()
def reload():
    """
    API call to model.warmup(). Just reload the model from /app/model/
    :return: None
    """
    asl_rec = als_rec.warmup()

if __name__ == '__main__':
    app()


# -*- coding: utf-8 -*-
import sys

n_corr_tok = 0
n_tok = 0

n_sent = 0
n_corr_sent = 0

sent_true = True

for line in sys.stdin:
    elems = line.strip().split('\t')
    if len(elems) != 3:
        n_sent += 1

        if sent_true:
            n_corr_sent += 1

        sent_true = True
        print('\n')
        continue

    print(line.strip())
    token = elems[0]
    true = elems[1]
    pred = elems[2]

    n_tok += 1
    if true == pred:
        n_corr_tok += 1
    else:
        sent_true = False

print('{}/{}'.format(n_corr_tok, n_tok))
print('{}/{}'.format(n_corr_sent, n_sent))


"""Given a m by n matrix, return all elements of the matrix in spiral order.

Time: O(m*n)
Space: O(1)
"""

def spiral_order(A):
    result = []

    if not A:
        return result

    left, right, top, bottom = 0, len(A[0]) - 1, 0, len(A) - 1

    while left <= right and top <= bottom:
        # go from left to right, starting from leftmost column
        for j in range(left, right + 1):
            result.append(A[top][j])

        # top to bottom, starting from first row
        for i in range(top + 1, bottom):
            result.append(A[i][right])

        # right to left, starting from rightmost column
        for j in reversed(range(left, right + 1)):
            if top < bottom:
                result.append(A[bottom][j])

        # bottom to top, starting from last row
        for i in reversed(range(top + 1, bottom)):
            if left < right:
                result.append(A[i][left])

        left, right, top, bottom = left + 1, right - 1, top + 1, bottom - 1

    return result

"""Another idea:
1. Extract first row from matrix and add to result list
2. Transpose the matrix, then flip it upside down (same as reversing it)
3. Repeat while matrix is not empty
"""

def spiral_order(A):
    result = []
    while A:
        first_row = A.pop(0)
        result.extend(first_row)
        # zip(*A) transposes matrix A
        A = [*zip(*A)][::-1]
    return result


class Solution:
    def merge(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        l = 0
        for i in range(m + n):
            if nums1[i] == 0:
                if l < n:
                    nums1[i] += nums2[l]
                l+=1
        return nums1.sort()


'''In this task, we’re going to be simulating an email message. Some of the logic has
been filled out for you in the email.py file.
    - Open the file called email.py.
    - Create a class definition for an Email which has four variables:
    has_been_read, email_contents, is_spam and from_address.
    - The constructor should initialise the sender’s email address.
    - The constructor should also initialise has_been_read and is_spam to false.
    - Create a function in this class called mark_as_read which should change
    has_been_read to true.
    - Create a function in this class called mark_as_spam which should change is_spam to true.
    - Create a list called inbox to store all emails (note that you can have a list of objects).
    - Then create the following functions:
        - add_email - which takes in the contents and email address from the
        received email to make a new Email object.
        - get_count - returns the number of messages in the store.
        - get_email - returns the contents of an email in the list. For this, allow
        the user to input an index i.e. get_email(i) returns the email stored
        at position i in the list. Once this has been done, has_been_read
        should now be true.
        - get_unread_emails - should return a list of all the emails that haven’t
        been read.
        - get_spam_emails - should return a list of all the emails that have been
        marked as spam.
        - delete - deletes an email in the inbox.
Now that you have these set up, let’s get everything working!
    - Fill in the rest of the logic for what should happen when the user inputs
    send/read/quit etc. so that all of the functions are utilised.. Some of it has
    been done for you.'''

class Email:
    # constructor for email object
    def __init__(self, from_address, email_contents):
        # initializes has_been_read as False
        self.has_been_read = False
        # stores email contents
        self.email_contents = email_contents
        # initializes is_spam as False
        self.is_spam = False
        # stores sender's email address
        self.from_address = from_address

    # function to mark email as read
    def mark_as_read(self):
        self.has_been_read = True

    # function to mark email as spam
    def mark_as_spam(self):
        self.is_spam = True

# list to store all email objects
inbox = []

# function to add email to inbox
def add_email(from_address, email_contents):
    # creates email object
    email = Email(from_address, email_contents)
    # adds email to inbox list
    inbox.append(email)

# function to get the number of emails in inbox
def get_count():
    return len(inbox)

# function to get email contents for a given index
def get_email(index):
    # checks if index is valid
    if index < 0 or index >= len(inbox):
        return None
    # marks email as read
    email = inbox[index]
    email.mark_as_read()
    # returns email contents
    return email.email_contents

# function to get unread emails
def get_unread_emails():
    # returns a list of unread email contents
    return [email.email_contents for email in inbox if not email.has_been_read]

# function to get spam emails
def get_spam_emails():
    # returns a list of spam email contents
    return [email.email_contents for email in inbox if email.is_spam]

# function to delete email at a given index
def delete(index):
    # checks if index is valid
    if index >= 0 and index < len(inbox):
        del inbox[index]

# variable to store user choice
user_choice = ""
# loop until user quits
while user_choice != "quit":
    # get user choice
    user_choice = input("What would you like to do - read/mark spam/send/quit?").lower()
    # action for read option
    if user_choice == "read":
        # check if inbox is empty
        if not inbox:
            print("No emails in inbox.")
        else:
            # iterate over emails in inbox
            for i, email in enumerate(inbox):
                # check if email has been read
                if not email.has_been_read:
                    print("{}. From: {}\n{}".format(i, email.from_address, email.email_contents))
    elif user_choice == "mark spam":
        # Check if there are any emails in the inbox
        if not inbox:
            print("No emails in inbox.")
        else:
            # Ask for the email index to mark as spam
            index = int(input("Enter the email index you want to mark as spam: "))
            # Check if the index is valid
            if index < 0 or index >= len(inbox):
                print("Invalid index.")
            else:
                # Mark the email as spam
                inbox[index].mark_as_spam()
                print("Email marked as spam.")
    elif user_choice == "send":
        # Ask for the sender's email address
        from_address = input("Enter the sender's email address: ")
        # Ask for the email contents
        email_contents = input("Enter the email contents: ")
        # Add the email to the inbox
        add_email(from_address, email_contents)
        print("Email added to inbox.")
    # user choice quit
    elif user_choice == "quit":
        print("Goodbye")
    else:
        print("Oops - incorrect input")


import os

def float_input( comentario = str() ):
    while True:
        valor = input( comentario )
        try:
            return float( valor )
        except:
            clear_window()
            print("Ingrese un valor numérico...")

def clear_window():
    os.system( 'cls' if os.name == 'nt' else 'clear' )


import logging
import torch
import argparse
import time
import multiprocessing
import os
import numpy as np
import math

from torch.utils.tensorboard import SummaryWriter
from torch.utils.data import DataLoader, SequentialSampler, RandomSampler
from torch.utils.data.distributed import DistributedSampler
from transformers import AdamW, get_linear_schedule_with_warmup
from tqdm import tqdm

from configs import add_args, set_dist, set_seed, set_hyperparas
from models import bulid_or_load_gen_model,bulid_or_load_cls_model
from utils import get_filenames, get_elapse_time, load_and_cache_gen_data, load_and_cache_defect_data,load_and_cache_clone_data, get_lang_by_task
from evaluator import smooth_bleu
from evaluator.CodeBLEU import calc_code_bleu
from evaluator.bleu import _bleu
import sys
from sklearn.metrics import recall_score, precision_score, f1_score
from tree_sitter import Language, Parser
from utils import retrieve2file
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

def evaluate_cls(args, model, eval_examples, eval_data, write_to_pred=False):
    eval_sampler = SequentialSampler(eval_data)
    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.dev_batch_size)

    # Eval!
    if write_to_pred == False:
        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Num batches = %d", len(eval_dataloader))
        logger.info("  Batch size = %d", args.dev_batch_size)
    eval_loss = 0.0
    nb_eval_steps = 0
    model.eval()
    logits = []
    labels = []
    for batch in tqdm(eval_dataloader, total=len(eval_dataloader), desc="Evaluating"):
        if args.sub_task == "POJ":
            inputs = batch[0].to(args.device)
            p_inputs = batch[1].to(args.device)
            n_inputs = batch[2].to(args.device)
            label = batch[3].to(args.device)
        else:
            inputs = batch[0].to(args.device) # inputs shape: [batch_size, args.max_source_length+args.max_target_length]
            label = batch[1].to(args.device) # label shape: [batch_size]
        with torch.no_grad():
            if args.sub_task == "POJ":
                lm_loss, logit = model(inputs,p_inputs,n_inputs,label)
            else:
                lm_loss, logit = model(inputs, label) # logit shape:[batch_size]

            eval_loss += lm_loss.mean().item()
            logits.append(logit.cpu().numpy())#logit shape: [batch_size]
            labels.append(label.cpu().numpy())#label shape: [batch_size]
        nb_eval_steps += 1
    logits = np.concatenate(logits, 0)
    labels = np.concatenate(labels, 0)
    if args.model_name == "unixcoder" and args.task == "clone":
        preds = logits > 0.5
    else:
        preds = logits[:, 1] > 0.5
    if args.task == 'defect':
        eval_acc = np.mean(labels == preds)
        eval_loss = eval_loss / nb_eval_steps
        perplexity = torch.tensor(eval_loss)

        result = {
            "eval_loss": float(perplexity),
            "eval_acc": round(eval_acc, 4) * 100,
        }
    elif args.task == 'clone':
        recall = recall_score(labels, preds)
        precision = precision_score(labels, preds)
        f1 = f1_score(labels, preds)
        result = {
            "eval_recall": float(recall) * 100,
            "eval_precision": float(precision) * 100,
            "eval_f1": float(f1) * 100,
            "eval_threshold": 0.5,
        }
    if write_to_pred == False:
        logger.info("***** Eval results *****")
        for key in sorted(result.keys()):
            logger.info("  %s = %s", key, str(round(result[key], 4)))

    if write_to_pred:
        with open(os.path.join(args.output_dir, "predictions.txt"), 'w') as f:
            for example, pred in zip(eval_examples, preds):
                if args.task == 'defect':
                    if args.model_name == "unixcoder":
                        if pred:
                            f.write(str(example.idx) + '\t1\n')
                        else:
                            f.write(str(example.idx) + '\t0\n')
                    else:
                        if pred:
                            f.write(str(example.idx) + '\t1\n')
                        else:
                            f.write(str(example.idx) + '\t0\n')
                elif args.task == 'clone':
                    if pred:
                        f.write(example.url1 + '\t' + example.url2 + '\t' + '1' + '\n')
                    else:
                        f.write(example.url1 + '\t' + example.url2 + '\t' + '0' + '\n')

    return result

def eval_ppl_epoch(args, eval_data, eval_examples, model, tokenizer):
    eval_sampler = SequentialSampler(eval_data)
    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.dev_batch_size,
                                 num_workers=4, pin_memory=True)
    # Start evaluating model
    logger.info("  " + "***** Running ppl evaluation *****")
    logger.info("  Num examples = %d", len(eval_examples))
    logger.info("  Batch size = %d", args.dev_batch_size)

    model.eval()
    eval_loss, batch_num = 0, 0
    for batch in tqdm(eval_dataloader, total=len(eval_dataloader), desc="Eval ppl"):
        batch = tuple(t.to(args.device) for t in batch)
        source_ids, target_ids = batch
        source_mask = source_ids.ne(tokenizer.pad_token_id)
        target_mask = target_ids.ne(tokenizer.pad_token_id)

        with torch.no_grad():
            if args.model_name in ['roberta', 'codebert', 'graphcodebert']:
#                     loss, _, _, attention = model(source_ids=source_ids, source_mask=source_mask,
#                                           target_ids=target_ids, target_mask=target_mask)
                loss, _, _, _ = model(source_ids=source_ids, source_mask=source_mask,
                                    target_ids=target_ids, target_mask=target_mask)
                if args.n_gpu > 1:
                    loss = loss.mean()
                eval_loss += loss.item()
                batch_num += 1
            elif args.model_name in ['unixcoder']:
                _,loss,num = model(source_ids=source_ids,target_ids=target_ids)
                if args.n_gpu > 1:
                    loss = loss.mean()
                eval_loss += loss.sum().item()
                batch_num += num.sum().item()
            else:
                outputs = model(input_ids=source_ids, attention_mask=source_mask,
                                labels=target_ids, decoder_attention_mask=target_mask)
                if isinstance(outputs,dict):
                    loss=outputs['loss']
                else:
                    loss = outputs.loss
                if args.n_gpu > 1:
                    loss = loss.mean()
                eval_loss += loss.item()
                batch_num += 1

    eval_loss = eval_loss / batch_num
    eval_ppl = round(np.exp(eval_loss), 5)
    return eval_ppl

def eval_bleu_epoch(args, eval_data, eval_examples, model, tokenizer, split_tag, criteria):
    logger.info(
        "  ***** Running bleu evaluation on {} data*****".format(split_tag))
    logger.info("  Num examples = %d", len(eval_examples))
    if split_tag == 'dev':
        batch_size = args.dev_batch_size
    elif split_tag == 'test':
        batch_size = args.test_batch_size
    else:
        batch_size = args.batch_size
    logger.info("  Batch size = %d", batch_size)
    eval_sampler = SequentialSampler(eval_data)
    if args.data_num == -1:
        eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=batch_size,
                                     num_workers=4, pin_memory=True)
    else:
        eval_dataloader = DataLoader(
            eval_data, sampler=eval_sampler, batch_size=batch_size)

    model.eval()
    pred_ids = []
    bleu, codebleu = 0.0, 0.0
    for batch in tqdm(eval_dataloader, total=len(eval_dataloader), desc="Eval bleu for {} set".format(split_tag)):
        source_ids = batch[0].to(args.device) #shape: (batch_size, max_source_len)
        source_mask = source_ids.ne(tokenizer.pad_token_id)
        if hasattr(model, 'module'):
            model = model.module # extract the model from the DataParallel wrapper
        with torch.no_grad():
            if args.model_name in ['roberta', 'codebert', 'graphcodebert']:
                preds, _ = model(source_ids=source_ids,
                                 source_mask=source_mask)
                top_preds = [pred[0].cpu().numpy() for pred in preds]
            elif args.model_name in ['unixcoder']:
                preds = model(source_ids=source_ids)  # preds shape: [batch_size, self.beam_size, max_target_len]
                top_preds = [pred[0].cpu().numpy() for pred in preds]# top_preds shape: batch_size * [max_target_len]
            else:
                preds = model.generate(source_ids,
                                       attention_mask=source_mask,
                                       use_cache=True,
                                       num_beams=args.beam_size,
                                       early_stopping=args.task == 'summarize',
                                       max_length=args.max_target_length)
                top_preds = list(preds.cpu().numpy())
            pred_ids.extend(top_preds)
    # pdb.set_trace()
    pred_nls = [tokenizer.decode(
        id, skip_special_tokens=True, clean_up_tokenization_spaces=False) for id in pred_ids]
    # unixcoder in fewshot will generate '\n' in small batch, and gradually disappear
    if args.model_name in ['unixcoder']:
        pred_nls = [id.replace('\n',' ').replace("        "," ").replace("    "," ").replace("\t"," ") for id in pred_nls]
    output_fn = os.path.join(args.res_dir, "test_{}.output".format(criteria))
    gold_fn = os.path.join(args.res_dir, "test_{}.gold".format(criteria))
    src_fn = os.path.join(args.res_dir, "test_{}.src".format(criteria))

    if args.task in ['defect']:
        target_dict = {0: 'false', 1: 'true'}
        golds = [target_dict[ex.target] for ex in eval_examples]
        eval_acc = np.mean([int(p == g) for p, g in zip(pred_nls, golds)])
        result = {'em': eval_acc, 'bleu': 0, 'codebleu': 0}

        with open(output_fn, 'w',encoding='utf-8') as f, open(gold_fn, 'w',encoding='utf-8') as f1, open(src_fn, 'w',encoding='utf-8') as f2:
            for pred_nl, gold in zip(pred_nls, eval_examples):
                f.write(pred_nl.strip() + '\n')
                f1.write(target_dict[gold.target] + '\n')
                f2.write(gold.source.strip() + '\n')
            logger.info("Save the predictions into %s", output_fn)
    else:
        dev_accs, predictions = [], []
        with open(output_fn, 'w',encoding='utf-8') as f, open(gold_fn, 'w',encoding='utf-8') as f1, open(src_fn, 'w',encoding='utf-8') as f2:
            for pred_nl, gold in zip(pred_nls, eval_examples):
                dev_accs.append(pred_nl.strip() == gold.target.strip())
                if args.task in ['summarize']:
                    predictions.append(str(gold.idx) + '\t' + pred_nl)
                    f.write(str(gold.idx) + '\t' +
                            pred_nl.strip().encode('utf8').decode() + '\n')
                    f1.write(str(gold.idx) + '\t' +
                             gold.target.strip().encode('utf8').decode() + '\n')
                    f2.write(str(gold.idx) + '\t' +
                             gold.source.strip().encode('utf8').decode() + '\n')
                else:
                    f.write(pred_nl.strip().encode('utf8').decode() + '\n')
                    f1.write(gold.target.strip().encode(
                        'utf8').decode() + '\n')
                    f2.write(gold.source.strip().encode(
                        'utf8').decode() + '\n')
        if args.task in ['summarize']:
            (goldMap, predictionMap) = smooth_bleu.computeMaps(predictions, gold_fn)
            bleu = round(smooth_bleu.bleuFromMaps(
                goldMap, predictionMap)[0], 2)
        else:
            bleu = round(_bleu(gold_fn, output_fn), 2)
            if split_tag == 'test' and args.task in ['refine', 'translate', 'generate' , 'clone']:
                args.lang = get_lang_by_task(args.task, args.sub_task)
                codebleu = calc_code_bleu.get_codebleu(
                    gold_fn, output_fn, args.lang,args=args)
        # except:
        #     bleu = 0.0
        #     codebleu = 0.0

        em = np.mean(dev_accs) * 100
        result = {'em': em, 'bleu': bleu}
        if not args.task == 'summarize' and split_tag == 'test':
            result['codebleu'] = codebleu * 100

    logger.info("***** Eval results *****")
    for key in sorted(result.keys()):
        logger.info("  %s = %s", key, str(round(result[key], 4)))

    return result

def main():
    parser = argparse.ArgumentParser()
    args = add_args(parser)
    t0 = time.time()

    set_dist(args)
    set_seed(args)
    set_hyperparas(args)

    logger.info(args)
    logger.info("************* args: *************")
    logger.info("args.model_name: "+str(args.model_name))
    logger.info("args.few_shot: "+str(args.few_shot))
    logger.info("args.task: "+str(args.task))
    logger.info("args.sub_task: "+str(args.sub_task))
    logger.info("*********************************")

    if args.task in ['summarize', 'translate', 'refine', 'generate','complete']:
        config, model, tokenizer = bulid_or_load_gen_model(args)
    elif args.task in ['defect','clone']:
        config, model, tokenizer = bulid_or_load_cls_model(args)
    model.to(args.device)
    if args.n_gpu > 1:
        model = torch.nn.DataParallel(model)
    pool = multiprocessing.Pool(args.cpu_count)
    args.train_filename, args.dev_filename, args.test_filename = get_filenames(
        args.data_dir, args.task, args.sub_task)
    fa = open(os.path.join(args.output_dir, 'summary.log'), 'a+',encoding='utf-8')

    if args.do_train:
        if args.local_rank in [-1, 0] and args.data_num == -1:
            summary_fn = '{}/{}'.format(args.summary_dir,
                                        '/'.join(args.output_dir.split('/')[1:]))
            tb_writer = SummaryWriter(summary_fn)
        # Prepare training data loader
        if args.task in ['summarize', 'translate', 'refine', 'generate','complete']:
            train_examples, train_data = load_and_cache_gen_data(
                args, args.train_filename, pool, tokenizer, 'train')
        elif args.task in ['defect']:
            train_examples, train_data = load_and_cache_defect_data(args, args.train_filename, pool, tokenizer, 'train')
        elif args.task in ['clone']:
            train_examples, train_data = load_and_cache_clone_data(args, args.train_filename, pool, tokenizer, 'train')
        if 0 and args.prefix_tuning:
            if args.retriever_mode in ['random', 'retrieve']:
                retrieve2file(args, train_examples, train_data)
        elif args.retriever_mode == 'old':
            args.prefix_dir = args.old_prefix_dir

        train_sampler = RandomSampler(
            train_data) if args.local_rank == -1 else DistributedSampler(train_data)
        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.batch_size,
                                      num_workers=4, pin_memory=True)
        # Prepare optimizer and schedule (linear warmup and decay)
        no_decay = ['bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
             'weight_decay': args.weight_decay},
            {'params': [p for n, p in model.named_parameters() if any(
                nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
        optimizer = AdamW(optimizer_grouped_parameters,
                          lr=args.lr, eps=args.adam_epsilon)
        num_train_optimization_steps = args.num_train_epochs * \
            len(train_dataloader)
        save_steps = max(len(train_dataloader), 1)
        if not args.is_clone_sample and args.task in ['clone']:
            save_steps = save_steps//20
        if args.task in ['defect']:
            save_steps = save_steps//1#4
        scheduler = get_linear_schedule_with_warmup(optimizer,
                                                    num_warmup_steps=int(args.warmup_steps) if args.warmup_steps >= 1 else num_train_optimization_steps * args.warmup_steps,
                                                    num_training_steps=num_train_optimization_steps)

        # Start training
        train_example_num = len(train_data)
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", train_example_num)
        logger.info("  Batch size = %d", args.batch_size)
        logger.info("  Batch num = %d", math.ceil(
            train_example_num / args.batch_size))
        logger.info("  Num epoch = %d", args.num_train_epochs)

        if args.task in ['summarize', 'translate', 'refine', 'generate','complete']:
            dev_dataset = {}
            global_step, best_bleu_em, best_ppl = 0, -1, 1e6
            not_loss_dec_cnt, not_bleu_em_inc_cnt = 0, 0 if args.do_eval_bleu else 1e6

            for cur_epoch in range(args.start_epoch, int(args.num_train_epochs)):
                bar = tqdm(train_dataloader, total=len(
                    train_dataloader), desc="Training")
                nb_tr_examples, nb_tr_steps, tr_loss = 0, 0, 0
                model.train()
                # kkk=0
                for step, batch in enumerate(bar):
                    # if kkk<2:
                    #     kkk+=1
                    # else:
                    #     break
                    batch = tuple(t.to(args.device) for t in batch)

                    source_ids, target_ids = batch
                    source_mask = source_ids.ne(tokenizer.pad_token_id)
                    target_mask = target_ids.ne(tokenizer.pad_token_id)

                    if args.model_name in ['roberta', 'codebert', 'graphcodebert']:
    #                     loss, _, _, attention = model(source_ids=source_ids, source_mask=source_mask,
    #                                           target_ids=target_ids, target_mask=target_mask)
                        loss, _, _, _ = model(source_ids=source_ids, source_mask=source_mask,
                                            target_ids=target_ids, target_mask=target_mask)
                    elif args.model_name in ['unixcoder']:
                        loss,_,_ = model(source_ids=source_ids,target_ids=target_ids)#mask strategy (e2d) written in Seq2seq
                    else:
                        outputs = model(input_ids=source_ids, attention_mask=source_mask,
                                        labels=target_ids, decoder_attention_mask=target_mask)
                        #attention =outputs.encoder_attentions
                        if isinstance(outputs,dict):
                            loss=outputs['loss']
                        else:
                            loss = outputs.loss

                    if args.n_gpu > 1:
                        loss = loss.mean()  # mean() to average on multi-gpu.
                    if args.gradient_accumulation_steps > 1:
                        loss = loss / args.gradient_accumulation_steps
                    tr_loss += loss.item()

                    nb_tr_examples += source_ids.size(0)
                    nb_tr_steps += 1
                    loss.backward()

                    if nb_tr_steps % args.gradient_accumulation_steps == 0:
                        # Update parameters
                        optimizer.step()
                        optimizer.zero_grad()
                        scheduler.step()
                        global_step += 1
                        train_loss = round(
                            tr_loss * args.gradient_accumulation_steps / (nb_tr_steps + 1), 4)
                        bar.set_description("[{}] Train loss {}".format(
                            cur_epoch, round(train_loss, 3)))
                        if args.data_num == -1:
                            tb_writer.add_scalar('train_loss_per_epoch', round(train_loss, 3), cur_epoch)
                            tb_writer.add_scalar('train_loss_per_step', round(train_loss, 3), global_step)
                    # result = {'epoch': cur_epoch,
                    #         'global_step': global_step, 'train_loss': round(train_loss, 3)}
                    # for key in sorted(result.keys()):
                    #     logger.info("  %s = %s", key, str(result[key]))

                if args.do_eval:
                    # Eval model with dev dataset
                    if 'dev_loss' in dev_dataset:
                        eval_examples, eval_data = dev_dataset['dev_loss']
                    else:
                        eval_examples, eval_data = load_and_cache_gen_data(
                            args, args.dev_filename, pool, tokenizer, 'dev')
                        dev_dataset['dev_loss'] = eval_examples, eval_data

                    eval_ppl = eval_ppl_epoch(
                        args, eval_data, eval_examples, model, tokenizer)
                    result = {'epoch': cur_epoch,
                            'global_step': global_step, 'eval_ppl': eval_ppl}
                    for key in sorted(result.keys()):
                        logger.info("  %s = %s", key, str(result[key]))
                    logger.info("  " + "*" * 20)
                    if args.data_num == -1:
                        tb_writer.add_scalar('dev_ppl', eval_ppl, cur_epoch)

                    # save last checkpoint
                    if args.save_last_checkpoints:
                        last_output_dir = os.path.join(
                            args.output_dir, 'checkpoint-last')
                        if not os.path.exists(last_output_dir):
                            os.makedirs(last_output_dir)
                        model_to_save = model.module if hasattr(
                            model, 'module') else model
                        output_model_file = os.path.join(
                            last_output_dir, "pytorch_model.bin")
                        torch.save(model_to_save.state_dict(), output_model_file)
                        logger.info("Save the last model into %s",
                                    output_model_file)

                    if eval_ppl < best_ppl:
                        not_loss_dec_cnt = 0
                        logger.info("  Best ppl:%s", eval_ppl)
                        logger.info("  " + "*" * 20)
                        fa.write("[%d] Best ppl changed into %.4f\n" %
                                (cur_epoch, eval_ppl))
                        best_ppl = eval_ppl

                        # Save best checkpoint for best ppl
                        output_dir = os.path.join(
                            args.output_dir, 'checkpoint-best-ppl')
                        if not os.path.exists(output_dir):
                            os.makedirs(output_dir)
                        if args.always_save_model:
                            model_to_save = model.module if hasattr(
                                model, 'module') else model
                            output_model_file = os.path.join(
                                output_dir, "pytorch_model.bin")
                            torch.save(model_to_save.state_dict(),
                                    output_model_file)
                            logger.info(
                                "Save the best ppl model into %s", output_model_file)
                    else:
                        not_loss_dec_cnt += 1
                        logger.info(
                            "Ppl does not decrease for %d epochs", not_loss_dec_cnt)
                        if all([x > args.patience for x in [not_bleu_em_inc_cnt, not_loss_dec_cnt]]):
                            early_stop_str = "[%d] Early stop as not_bleu_em_inc_cnt=%d, and not_loss_dec_cnt=%d\n" % (
                                cur_epoch, not_bleu_em_inc_cnt, not_loss_dec_cnt)
                            logger.info(early_stop_str)
                            fa.write(early_stop_str)
                            break
                    logger.info("***** CUDA.empty_cache() *****")
                    torch.cuda.empty_cache()
                    if args.do_eval_bleu:
                        eval_examples, eval_data = load_and_cache_gen_data(args, args.dev_filename, pool, tokenizer, 'dev',
                                                                        only_src=True, is_sample=True)

                        result = eval_bleu_epoch(
                            args, eval_data, eval_examples, model, tokenizer, 'dev', 'e%d' % cur_epoch)
                        dev_bleu, dev_em = result['bleu'], result['em']
                        if args.task in ['summarize']:
                            dev_bleu_em = dev_bleu
                        elif args.task in ['defect']:
                            dev_bleu_em = dev_em
                        else:
                            dev_bleu_em = dev_bleu + dev_em
                        if args.data_num == -1:
                            tb_writer.add_scalar(
                                'dev_bleu_em', dev_bleu_em, cur_epoch)
                            # tb_writer.add_scalar('dev_em', dev_em, cur_epoch)
                        if dev_bleu_em > best_bleu_em:
                            not_bleu_em_inc_cnt = 0
                            logger.info("  [%d] Best bleu+em: %.2f (bleu: %.2f, em: %.2f)",
                                        cur_epoch, dev_bleu_em, dev_bleu, dev_em)
                            logger.info("  " + "*" * 20)
                            best_bleu_em = dev_bleu_em
                            fa.write("[%d] Best bleu+em changed into %.2f (bleu: %.2f, em: %.2f)\n" % (
                                cur_epoch, best_bleu_em, dev_bleu, dev_em))
                            # Save best checkpoint for best bleu
                            output_dir = os.path.join(
                                args.output_dir, 'checkpoint-best-bleu')
                            if not os.path.exists(output_dir):
                                os.makedirs(output_dir)
                            if args.data_num == -1 or args.always_save_model:
                                model_to_save = model.module if hasattr(
                                    model, 'module') else model
                                output_model_file = os.path.join(
                                    output_dir, "pytorch_model.bin")
                                torch.save(model_to_save.state_dict(),
                                        output_model_file)
                                logger.info(
                                    "Save the best bleu model into %s", output_model_file)
                        else:
                            not_bleu_em_inc_cnt += 1
                            logger.info(
                                "Bleu_em does not increase for %d epochs", not_bleu_em_inc_cnt)
                            if all([x > args.patience for x in [not_bleu_em_inc_cnt, not_loss_dec_cnt]]):
                                stop_early_str = "[%d] Early stop as not_bleu_em_inc_cnt=%d, and not_loss_dec_cnt=%d\n" % (
                                    cur_epoch, not_bleu_em_inc_cnt, not_loss_dec_cnt)
                                logger.info(stop_early_str)
                                fa.write(stop_early_str)
                                break
                logger.info("***** CUDA.empty_cache() *****")
                torch.cuda.empty_cache()

        elif args.task in ['defect']:
            global_step, best_acc = 0, 0
            not_acc_inc_cnt = 0
            is_early_stop = False
            for cur_epoch in range(args.start_epoch, int(args.num_train_epochs)):
                bar = tqdm(train_dataloader, total=len(train_dataloader), desc="Training")
                nb_tr_examples, nb_tr_steps, tr_loss = 0, 0, 0
                model.train()
                for step, batch in enumerate(bar):
                    if args.sub_task == "POJ":
                        source_ids = batch[0].to(args.device)
                        p_inputs = batch[1].to(args.device)
                        n_inputs = batch[2].to(args.device)
                        labels = batch[3].to(args.device)
                        model.train()
                        loss,vec = model(source_ids,p_inputs,n_inputs,labels)
                    else:
                        if args.model_name == 'unixcoder':
                            source_ids = batch[0].to(args.device)
                            labels = batch[1].to(args.device)
                            loss, logits = model(source_ids, labels)
                        elif args.model_name in ['t5','codet5','bart','plbart']:
                            batch = tuple(t.to(args.device) for t in batch)
                            source_ids, labels = batch
                            outputs = model(source_ids, labels)
                            if isinstance(outputs,dict):
                                loss=outputs['loss']
                            else:
                                loss, logits = outputs
                        else:
                            batch = tuple(t.to(args.device) for t in batch)
                            source_ids, labels = batch
                            loss, logits = model(source_ids, labels)

                    if args.n_gpu > 1:
                        loss = loss.mean()  # mean() to average on multi-gpu.
                    if args.gradient_accumulation_steps > 1:
                        loss = loss / args.gradient_accumulation_steps
                    tr_loss += loss.item()

                    nb_tr_examples += source_ids.size(0)
                    nb_tr_steps += 1
                    loss.backward()
                    # torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

                    if nb_tr_steps % args.gradient_accumulation_steps == 0:
                        # Update parameters
                        optimizer.step()
                        optimizer.zero_grad()
                        scheduler.step()
                        global_step += 1
                        train_loss = round(tr_loss * args.gradient_accumulation_steps / nb_tr_steps, 4)
                        bar.set_description("[{}] Train loss {}".format(cur_epoch, round(train_loss, 3)))
                        if args.data_num == -1:
                            tb_writer.add_scalar('train_loss_per_epoch', round(train_loss, 3), cur_epoch)
                            tb_writer.add_scalar('train_loss_per_step', round(train_loss, 3), global_step)

                    if (step + 1) % save_steps == 0 and args.do_eval:
                        logger.info("***** CUDA.empty_cache() *****")
                        torch.cuda.empty_cache()

                        eval_examples, eval_data = load_and_cache_defect_data(args, args.dev_filename, pool, tokenizer,
                                                                            'dev',is_sample=True)

                        result = evaluate_cls(args, model, eval_examples, eval_data)
                        eval_acc = result['eval_acc']

                        if args.data_num == -1:
                            tb_writer.add_scalar('dev_acc', round(eval_acc, 4), len(train_dataloader)//save_steps*cur_epoch+(step + 1) // save_steps-1)

                        # save last checkpoint
                        last_output_dir = os.path.join(args.output_dir, 'checkpoint-last')
                        if not os.path.exists(last_output_dir):
                            os.makedirs(last_output_dir)

                        if True or args.data_num == -1 and args.save_last_checkpoints:
                            model_to_save = model.module if hasattr(model, 'module') else model
                            output_model_file = os.path.join(last_output_dir, "pytorch_model.bin")
                            torch.save(model_to_save.state_dict(), output_model_file)
                            logger.info("Save the last model into %s", output_model_file)

                        if eval_acc > best_acc:
                            not_acc_inc_cnt = 0
                            logger.info("  Best acc: %s", round(eval_acc, 4))
                            logger.info("  " + "*" * 20)
                            fa.write("[%d] Best acc changed into %.4f\n" % (cur_epoch, round(eval_acc, 4)))
                            best_acc = eval_acc
                            # Save best checkpoint for best ppl
                            output_dir = os.path.join(args.output_dir, 'checkpoint-best-acc')
                            if not os.path.exists(output_dir):
                                os.makedirs(output_dir)
                            if args.data_num == -1 or True:
                                model_to_save = model.module if hasattr(model, 'module') else model
                                output_model_file = os.path.join(output_dir, "pytorch_model.bin")
                                torch.save(model_to_save.state_dict(), output_model_file)
                                logger.info("Save the best ppl model into %s", output_model_file)
                        else:
                            not_acc_inc_cnt += 1
                            logger.info("acc does not increase for %d epochs", not_acc_inc_cnt)
                            if not_acc_inc_cnt > args.patience*len(train_dataloader)//save_steps:
                                logger.info("Early stop as acc do not increase for %d times", not_acc_inc_cnt)
                                fa.write("[%d] Early stop as not_acc_inc_cnt=%d\n" % (cur_epoch, not_acc_inc_cnt))
                                is_early_stop = True
                                break

                    model.train()
                if is_early_stop:
                    break

                logger.info("***** CUDA.empty_cache() *****")
                torch.cuda.empty_cache()
        elif args.task in ['clone']:
            global_step, best_f1 = 0, 0
            not_f1_inc_cnt = 0
            is_early_stop = False
            for cur_epoch in range(args.start_epoch, int(args.num_train_epochs)):
                bar = tqdm(train_dataloader, total=len(train_dataloader), desc="Training")
                nb_tr_examples, nb_tr_steps, tr_loss = 0, 0, 0
                model.train()
                for step, batch in enumerate(bar):
                    if args.sub_task == "POJ":
                        source_ids = batch[0].to(args.device)
                        p_inputs = batch[1].to(args.device)
                        n_inputs = batch[2].to(args.device)
                        labels = batch[3].to(args.device)
                        loss,vec = model(source_ids,p_inputs,n_inputs,labels)
                    else:
                        if args.model_name == 'unixcoder':
                            source_ids = batch[0].to(args.device)
                            labels = batch[1].to(args.device)
                            loss, logits = model(source_ids, labels)
                        elif args.model_name in ['t5','codet5','bart','plbart']:
                            batch = tuple(t.to(args.device) for t in batch)
                            source_ids, labels = batch
                            outputs = model(source_ids, labels=labels)
                            if isinstance(outputs,dict):
                                loss=outputs['loss']
                            else:
                                loss, logits = outputs
                        else:
                            batch = tuple(t.to(args.device) for t in batch)#shape: (2*(batch_size, seq_len))
                            source_ids, labels = batch #[batch,1024] [batch]
                            loss, logits = model(source_ids, labels)

                    if args.n_gpu > 1:
                        loss = loss.mean()  # mean() to average on multi-gpu.
                    if args.gradient_accumulation_steps > 1:
                        loss = loss / args.gradient_accumulation_steps
                    tr_loss += loss.item()

                    nb_tr_examples += source_ids.size(0)
                    nb_tr_steps += 1
                    loss.backward()
                    # torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

                    if nb_tr_steps % args.gradient_accumulation_steps == 0:
                        # Update parameters
                        optimizer.step()
                        optimizer.zero_grad()
                        scheduler.step()
                        global_step += 1
                        train_loss = round(tr_loss * args.gradient_accumulation_steps / nb_tr_steps, 4)
                        bar.set_description("[{}] Train loss {}".format(cur_epoch, round(train_loss, 3)))
                        if args.data_num == -1:
                            tb_writer.add_scalar('train_loss_per_epoch', round(train_loss, 3), cur_epoch)
                            tb_writer.add_scalar('train_loss_per_step', round(train_loss, 3), global_step)

                    if (step + 1) % save_steps == 0 and args.do_eval:
                        logger.info("***** CUDA.empty_cache() *****")
                        torch.cuda.empty_cache()

                        eval_examples, eval_data = load_and_cache_clone_data(args, args.dev_filename, pool, tokenizer,
                                                                            'dev', is_sample=True)

                        result = evaluate_cls(args, model, eval_examples, eval_data)
                        eval_f1 = result['eval_f1']

                        if args.data_num == -1:
                            tb_writer.add_scalar('dev_f1', round(eval_f1, 4), len(train_dataloader)//save_steps*cur_epoch+(step + 1) // save_steps-1)

                        # save last checkpoint
                        last_output_dir = os.path.join(args.output_dir, 'checkpoint-last')
                        if not os.path.exists(last_output_dir):
                            os.makedirs(last_output_dir)

                        if True or args.data_num == -1 and args.save_last_checkpoints:
                            model_to_save = model.module if hasattr(model, 'module') else model
                            output_model_file = os.path.join(last_output_dir, "pytorch_model.bin")
                            torch.save(model_to_save.state_dict(), output_model_file)
                            logger.info("Save the last model into %s", output_model_file)

                        if eval_f1 > best_f1:
                            not_f1_inc_cnt = 0
                            logger.info("  Best f1: %s", round(eval_f1, 4))
                            logger.info("  " + "*" * 20)
                            fa.write("[%d] Best f1 changed into %.4f\n" % (cur_epoch, round(eval_f1, 4)))
                            best_f1 = eval_f1
                            # Save best checkpoint for best ppl
                            output_dir = os.path.join(args.output_dir, 'checkpoint-best-f1')
                            if not os.path.exists(output_dir):
                                os.makedirs(output_dir)
                            if args.data_num == -1 or True:
                                model_to_save = model.module if hasattr(model, 'module') else model
                                output_model_file = os.path.join(output_dir, "pytorch_model.bin")
                                torch.save(model_to_save.state_dict(), output_model_file)
                                logger.info("Save the best ppl model into %s", output_model_file)
                        else:
                            not_f1_inc_cnt += 1
                            logger.info("F1 does not increase for %d epochs", not_f1_inc_cnt)
                            if not_f1_inc_cnt > args.patience*len(train_dataloader)//save_steps:
                                logger.info("Early stop as f1 do not increase for %d times", not_f1_inc_cnt)
                                fa.write("[%d] Early stop as not_f1_inc_cnt=%d\n" % (cur_epoch, not_f1_inc_cnt))
                                is_early_stop = True
                                break

                    model.train()
                if is_early_stop:
                    break

                logger.info("***** CUDA.empty_cache() *****")
                torch.cuda.empty_cache()

        if args.local_rank in [-1, 0] and args.data_num == -1:
            tb_writer.close()
        logger.info("Finish training and take %s", get_elapse_time(t0))

    if args.do_test:

        logger.info("  " + "***** Testing *****")
        logger.info("  Batch size = %d", args.test_batch_size)
        if args.task in ['summarize', 'translate', 'refine', 'generate','complete']:
            for criteria in ['best-bleu', 'best-ppl']:  # 'best-bleu', 'best-ppl', 'last'
                file = os.path.join(
                    args.output_dir, 'checkpoint-{}/pytorch_model.bin'.format(criteria))
                logger.info("Reload model from {}".format(file))

                model = model.module if hasattr(model, 'module') else model
                model.load_state_dict(torch.load(file))
                # if args.n_gpu > 1:
                #     # multi-gpu training
                #     model = torch.nn.DataParallel(model)
                # model.load_state_dict(torch.load(file))
                eval_examples, eval_data = load_and_cache_gen_data(args, args.test_filename, pool, tokenizer, 'test',
                                                                only_src=True, is_sample=False)
                result = eval_bleu_epoch(
                    args, eval_data, eval_examples, model, tokenizer, 'test', criteria)
                test_bleu, test_em = result['bleu'], result['em']
                test_codebleu = result['codebleu'] if 'codebleu' in result else 0
                result_str = "[%s] bleu-4: %.2f, em: %.4f, codebleu: %.4f\n" % (
                    criteria, test_bleu, test_em, test_codebleu)
                logger.info("*********************************")
                logger.info(result_str)
                logger.info("*********************************")
                fa.write(result_str)
                if args.res_fn:
                    with open(args.res_fn, 'a+',encoding='utf-8') as f:
                        f.write('[Time: {}] {}\n'.format(
                            get_elapse_time(t0), file))
                        f.write(result_str)
        elif args.task in ['defect']:
            for criteria in ['best-acc']:
                file = os.path.join(args.output_dir, 'checkpoint-{}/pytorch_model.bin'.format(criteria))
                logger.info("Reload model from {}".format(file))

                model = model.module if hasattr(model, 'module') else model
                model.load_state_dict(torch.load(file))
                # if args.n_gpu > 1:
                #     # multi-gpu training
                #     model = torch.nn.DataParallel(model)

                eval_examples, eval_data = load_and_cache_defect_data(args, args.test_filename, pool, tokenizer, 'test',
                                                                    False)
                result = evaluate_cls(args, model, eval_examples, eval_data, write_to_pred=True)
                logger.info(" test_acc = %.4f" % ( result['eval_acc']))
                logger.info("*********************************")
                logger.info("[%s]  test_acc = %.4f" % (criteria, result['eval_acc']))
                logger.info("*********************************")

                fa.write("[%s] test-acc: %.4f\n" % (criteria, result['eval_acc']))
                if args.res_fn:
                    with open(args.res_fn, 'a+') as f:
                        f.write('[Time: {}] {}\n'.format(get_elapse_time(t0), file))
                        f.write("[%s] acc: %.4f\n\n" % (
                            criteria, result['eval_acc']))
        elif args.task in ['clone']:
            for criteria in ['best-f1']:
                file = os.path.join(args.output_dir, 'checkpoint-{}/pytorch_model.bin'.format(criteria))
                logger.info("Reload model from {}".format(file))

                model = model.module if hasattr(model, 'module') else model
                model.load_state_dict(torch.load(file))
                # if args.n_gpu > 1:
                #     # multi-gpu training
                #     model = torch.nn.DataParallel(model)
                # model.load_state_dict(torch.load(file))

                eval_examples, eval_data = load_and_cache_clone_data(args, args.test_filename, pool, tokenizer, 'test',
                                                                    False)

                result = evaluate_cls(args, model, eval_examples, eval_data, write_to_pred=True)

                logger.info("  test_f1 = %.4f", result['eval_f1'])
                logger.info("  test_prec = %.4f", result['eval_precision'])
                logger.info("  test_rec = %.4f", result['eval_recall'])
                logger.info("*********************************")
                logger.info("[%s] test-f1: %.4f, precision: %.4f, recall: %.4f\n" % (
                    criteria, result['eval_f1'], result['eval_precision'], result['eval_recall']))
                logger.info("*********************************")

                fa.write("[%s] test-f1: %.4f, precision: %.4f, recall: %.4f\n" % (
                    criteria, result['eval_f1'], result['eval_precision'], result['eval_recall']))
                if args.res_fn:
                    with open(args.res_fn, 'a+') as f:
                        f.write('[Time: {}] {}\n'.format(get_elapse_time(t0), file))
                        f.write("[%s] f1: %.4f, precision: %.4f, recall: %.4f\n\n" % (
                            criteria, result['eval_f1'], result['eval_precision'], result['eval_recall']))

    logger.info("Finish and take {}".format(get_elapse_time(t0)))
    fa.write("Finish and take {}".format(get_elapse_time(t0)))
    fa.close()

if __name__ == "__main__":
    main()