import time

import numpy as np
from matplotlib import pyplot as plt

from SimulationWorld import SimulationWorld,WorldAnimation
from SimulationRobot import SimulationRobot

if __name__ == "__main__":

    #This demo will run a simulation, save it to a file, load it from the file and then animate it.

    # You can change the behavoir of the robots in the simulation by editing Lines 96 - 100 to initialise the robot from a different dictionary

    # If you want to load a previous simulation set simulate world to False and edit line 168 to reflect your simulations file path

    #Simulate the world or load from the "world.pickle" file in this scripts directory
    simulate_world = True
    if simulate_world:
        #To create a simulation first create a SimulationWorld object
        world = SimulationWorld()

        #You can seed the pseudo random number generator before simulation to ensure it performs the same steps every time and reproduce bugs
        #np.random.seed(0)

        ########################   Random walking robot parameter dictionary   #########################
        #Random walking is a series of displacements and changes in heading. Each of these can be described with a certain propbablility distrobution leading to
        #different types of random motion such as brownian motion, levy walk and correlated random walks

        robot_params_rw = {  "algorithm"             : "random_walker",
                          "dir_change_distro"     : ("gaussian",0.0,0.5),
                          "step_len_distro"       : ("gaussian",0.0,0.1),
                          "max_speed"             : 1.0,
                          "radius"                : 0.1
                        }

        ########################   Boid flocking robot parameter dictionary   ########################
        #Renyolds flocking algorithm uses 3 rules to cause flocking behavour. This implimentation uses an additional rule which makes the robots head towards the centre
        #of the world based on their distance
        robot_params_boid = {  "algorithm"             : "boid_flocker",

                              "neighbourhood_mode"    : "distance",
                              "neighbourhood_distance": 3.0,

                              # "neighbourhood_mode"    : "nearist",
                              # "neighbourhood_size"    : 1.0,

                              "seperation_dist"       : 1.0,

                              "update_period"           : 0.1,

                              "cohesion_coefficient"    : 30.0,
                              "alignment_coefficient"   : 60.0,
                              "seperation_coefficient"  : 20.0,

                              "central_pull_coefficient": 30.0,

                              "rotational_p_control"  : 0.9,
                              "max_speed"             : 1.0,
                              "radius"                : 0.1
                            }
        ########################   Firefly inspired synchornisation robot parameter dictionary   ########################
        #These robots will change their state with a certain period (flash_period) but will start out of sync. By increasing their activation
        #value when they see nearby flashes they aim to synchronise their flashes (have the same phase)

        robot_params_firefly = {  "algorithm"             : "firefly_sync",

                                  "neighbourhood_mode"    : "distance",
                                  "neighbourhood_distance": 1.0,

                                  # "neighbourhood_mode"    : "nearist",
                                  # "neighbourhood_size"    : 20,

                                  "update_period"           : 0.1,

                                  "flash_period"                : 3.0,
                                  "flash_on_duration"           : 0.5,
                                  "activation_increase"         : 0.02,

                                  "static"                 : True,

                                  "dir_change_distro"     : ("uniform",-np.pi,np.pi),
                                  "step_len_distro"       : ("gaussian",0.0,0.2),

                                  "max_speed"             : 1.0,
                                  "radius"                : 0.1
                                }
        #Robots are created using the dictionaries above as it allows all the parameters of the robot to viewed in on data structure

        #r = SimulationRobot(robot_params_rw) # Uncommment to switch to random walking robots
        r = SimulationRobot(robot_params_boid)
        #r = SimulationRobot(robot_params_firefly) #Uncomment to switch to a firefly synchronisation demo

        #The required simulation time step  and collision bin size are then calculated. These robots currently go at fixed size but if there speed could vary the fasted possible velocity should be used here
        world.calculate_neighbours = True
        #This ensure a robot can't move through another robot during a single time step
        world.dt = r.robot_params["radius"]/r.robot_params["max_speed"]
        #This ensures no robot can cross into another colision bin during a single timestep
        world.bin_size = (r.robot_params["radius"]*2.0 + (r.robot_params["max_speed"])*world.dt+0.01)

        #Now that we've set the bin size we can initialise the collision bins
        world.robot_collisions = False
        world.init_physics()

        #Adds our robot to the world
        world.populate(100,r)

        #Initialises our robot's positions are the start  of the simulation, currently this is a the smallest possible square the robots can occuoy with 0.25m inbetween their bodies

        world.arrange(mode = "smallest_square",center_pos = (0.0,0.0),robot_separation = 0.1, added_noise = 0.0)
        #world.arrange(mode = "uniform_box",center_pos = (0.0,0.0),robot_separation = 0.1, added_noise = 0.0,box_size = (world.barriers[1]-world.barriers[0],world.barriers[3]-world.barriers[2])) #This will arrange the robots over the entire world area rarther than in the middle

        #The number of steps we will simulate
        #This is the time we want to simulate divided by the amount of time we simulate each time step (dt)
        steps_num = int(2*60.0/world.dt)

        #If will create a window that shows the simulation's state evert so often (dictated by snap_shot_steps) - useful for debugging
        plot_snap_shots = False
        snap_shot_steps = 0.2/world.dt

        #Execution time stats, how long on average a time_step takes to simulate and the longest time taken to simulate a time step
        exec_times = 0.0
        max_exec_time = 0.0
        sim_start_time = time.time()

        #Pre-allocates memory for data logging. Saving data every timestep can result in very large file sizes so we can opt to only do it every 2 timesteps
        #world.data_log_period = world.dt*2
        world.init_data_log(steps_num)

        if plot_snap_shots:
            world_anim_snapshot = WorldAnimation(world,fast_plot = True)
            plt.show(block = False)

        print("Starting simulation")
        print("Swarm Size : {:d} dt = {:4.2f} measurement dt : {:4.2f}".format(world.num_robots,world.dt,world.data_log_period))
        #Main simulation loop
        for step in range(steps_num-1):
            start_time = time.time()
            world.time_step()
            dt = (time.time() - start_time)
            exec_times+= dt
            if dt > max_exec_time:
                max_exec_time = dt

            if step%max((1,steps_num/100))==0:
                print("Simulating... {:4.2f}% ETA {:4.2f} mins".format(float(step)/steps_num*100.0,(steps_num-step-1)*(exec_times/(step+1))/60.0))
            if plot_snap_shots and step%snap_shot_steps == 0:
                world_anim_snapshot.plot_snapshot(step)
                plt.draw()
                plt.pause(0.01)

        print("Average execution time {:4.4f}s per timestep. Maximum time per timestep = {:4.4f}s Time taken {:4.2f}s".format(exec_times/float(steps_num),max_exec_time,time.time()-sim_start_time))

        #Saves the world using pythons pickle format to the current working directory (should be this scripts directory)
        world.save("world.pickle")

    world = SimulationWorld().load("world.pickle")
    #Plots the final state of the simulation and animates
    world_anim_snapshot = WorldAnimation(world)
    world_anim_snapshot.plot_snapshot(world.dt*world.t_step)
    plt.title("Final simulation state")

    robot_cmap =      { 0 : 'dimgrey',
                        1 : 'palegreen',
                        2 : 'lightcoral',
                        3 : 'blue'}

    world_anim_final = WorldAnimation(world,robot_trail_length = 0, robot_trail_width = 0.1,robot_state_cmap = robot_cmap, robot_labels = False, view_collision_bins = False, viewing_bounds = world.barriers,fast_plot = False )

    world_anim_final.start_animation(start_time = None,final_time = None,speed = 1.0,time_between_frames = None)

    #Simulations can be saved to .mp4 format but this requires certain codecs
    #world_anim_final.start_animation(save_path = "world_animation.mp4",start_time = None,final_time = None,speed = 1.0,time_between_frames = None)


lista_aprovados  = []
lista_reprovados = []

aluno = {}
sala  = {}
nota  = {}

adicionar = 'sim'

while adicionar == 'sim':
    aluno = input('Digite o nome do aluno: ')
    sala  = input('Digite o número ou letra da sala: ')
    nota  = int(input('Digite a nota: '))

    if  nota >= 50:
        lista_aprovados.extend([aluno, sala, nota])
    else:
        nota < 50
        lista_reprovados.extend([aluno, sala, nota])

    adicionar = input('Deseja adicionar mais uma nota? ')

print('Os alunos aprovados são:')
for aprovado in lista_aprovados:
    print(aprovado)

print('Os alunos reprovados são:')
for reprovado in lista_reprovados:
    print(reprovado)


'''
2021.09.01. 유기농 배추
https://www.acmicpc.net/problem/1012
'''
from sys import stdin
from collections import deque

stdin = open("../input.txt", "r")

dx = [1, -1, 0, 0]
dy = [0, 0, 1, -1]

tc = int(stdin.readline().strip())
for _ in range(tc):
    m, n, k = map(int, stdin.readline().strip().split())
    board = [[0] * m for _ in range(n)]
    vis = [[False] * m for _ in range(n)]
    q = deque()
    cnt = 0

    for _ in range(k):
        x, y = map(int, stdin.readline().strip().split())
        board[y][x] = 1

    for i in range(n):
        for j in range(m):
            if vis[i][j] or board[i][j] == 0: continue
            q.append([i, j])
            vis[i][j] = True
            cnt += 1

            while q:
                cur_y, cur_x = q.popleft()
                for dir in range(4):
                    ny, nx = cur_y + dy[dir], cur_x + dx[dir]
                    if ny < 0 or ny >= n or nx < 0 or nx >= m: continue
                    if vis[ny][nx] or board[ny][nx] == 0: continue
                    q.append([ny, nx])
                    vis[ny][nx] = True
    print(cnt)


# Aradiabot image glitching functions.
# Transcribed over from my 'fastglitch' repository.
from io import BytesIO, StringIO
import random, sys, PIL.Image, PIL.ImageChops, PIL.ImageDraw, os
chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890'
import asyncio

def genImg(fname):
	img = PIL.Image.open(fname)
	img = img.convert('RGBA')
	proto1 = RandomByteAddition(img, random.randint(1,16))
	proto2 = RGBOffset(proto1, random.randint(1,64))
	proto3 = PixelOffset(proto2, random.randint(1,512))
	proto4 = Artifact(proto3, random.randint(1,64))
	proto5 = RowSlice(proto4, random.randint(1,32))
	proto6 = Noise(proto5, random.randint(25000,50000))
	p = proto6.convert('RGB')
	p.save('new' + fname + '.png')
	proto1.close()
	proto2.close()
	proto3.close()
	proto4.close()
	proto5.close()
	proto6.close()

def RandomByteAddition(image, seed):
	bytesBroken = False
	bytesobj = BytesIO()
	image.save(bytesobj, 'jpeg')
	iter = seed
	bytesobj.seek(1024)
	if seed > 0:
		for x in range(0, iter):
			bytes2 = bytesobj
			bytesobj.seek(random.randint(0, 32), 1)
			byte = random.choice(chars)
			bytesobj.write(bytes(byte, 'utf-8'))
			try:
				PIL.Image.open(bytesobj)
			except:
				bytesBroken = True
				break

	if bytesBroken == True:
		bytes2.seek(0)
		new_img = PIL.Image.open(bytes2)
	else:
		bytesobj.seek(0)
		new_img = PIL.Image.open(bytesobj)
	return new_img

def RGBOffset(image, distance):
	distance = distance * 30
	r, g, b = image.split()
	r = PIL.ImageChops.offset(r, distance * -1, 0)
	b = PIL.ImageChops.offset(b, distance, 0)
	new_img = PIL.Image.merge('RGB', (r, g, b))
	return new_img

def PixelOffset(image, distance):
	new_img = PIL.ImageChops.offset(image, distance)
	return new_img

def RowSlice(image, sliceamount):
	cps = 0
	new_img = image
	for x in range(sliceamount):
		upbound = cps
		downbound = upbound + random.randint(16, 128)
		if downbound > image.height:
			break
		box = (0,
		 upbound,
		 new_img.width,
		 downbound)
		reigon = new_img.crop(box)
		distance = random.randint(-128, 128)
		reigon = PIL.ImageChops.offset(reigon, distance, 0)
		new_img.paste(reigon, box)
		reigon.close()
		cps = downbound

		return new_img

def Artifact(image, screwamount):
	tnspimg = image.convert('RGBA')
	base = PIL.Image.new('RGBA', tnspimg.size, (255, 255, 255, 0))
	rows = PIL.ImageDraw.Draw(base)
	cps = 0
	for x in range(screwamount):
		leftbound = cps
		rightbound = leftbound + random.randint(32, 128)
		if rightbound > image.width:
			break
		y1 = random.randint(0, image.height - int(round(image.height / 2.0, 0)))
		x1 = random.randint(leftbound, rightbound - 1)
		y2 = random.randint(y1, image.height)
		x2 = rightbound
		color = (random.randint(0, 255),
		 random.randint(0, 255),
		 random.randint(0, 255),
		 random.randint(64, 200))
		rows.rectangle((x1,
		 y1,
		 x2,
		 y2), fill=color)
		cps = rightbound

	new_img = PIL.Image.alpha_composite(tnspimg, base)
	return new_img

def Noise(image, pixels):
	for x in range(1, pixels):
		image.putpixel((random.randint(1, image.width - 1), random.randint(1, image.height - 1)), (random.randint(1, 255), random.randint(1, 255), random.randint(1, 255)))

	new_img = image
	return new_img


import gzip
import shutil
import os
import wget
import csv
import linecache
from shutil import copyfile
import ipywidgets as widgets
import numpy as np
import pandas as pd
import tweepy
import json_lines
import re
import json
from urllib.parse import urlparse
import requests

def EDA_data():
    all_path=[]
    for i in np.arange(64):
        if i<10:
            all_path.append("data/all_data_split/mydata_0"+str(i+1)+".pkl")
        else:
            all_path.append("data/all_data_split/mydata_"+str(i+1)+".pkl")
    all_path_in=[]
    for i in np.arange(64):
        if i<10:
            all_path_in.append("data/all_data_split/mydata_0"+str(i+1))
        else:
            all_path_in.append("data/all_data_split/mydata_"+str(i+1))

    for link in all_path_in:
        with open(link) as f:
            df = pd.DataFrame(json.loads(line) for line in f)
        df.to_pickle(link+".pkl")

    pattern_link='.*http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+.*'
    pattern_re='RT @.*'

    all_values=pd.DataFrame()
    for path in all_path:
        df = pd.read_pickle(path)

        med=df[['author_id']]
        df=0
        med['count']=med['author_id'].apply(lambda x: 1)
        med['count_2']=med['author_id'].apply(lambda x: 1)
        all_values[path]=med.groupby('author_id').sum().groupby('count').count()['count_2']
        print(path)

    all_values.fillna(0).apply(sum,axis=1).to_csv('same_author.csv')
    total_amount=all_values.fillna(0).apply(sum,axis=1).sum()

    P_re=0
    P_url=0
    authors=[]
    for path in all_path:
        print(path)
        all_values=pd.DataFrame()
        df = pd.read_pickle(path)
        all_values=all_values.append(df[['author_id','text','id']])
        series_a=df['text'].str.contains(pattern_re)
        series_b=df['text'].str.contains(pattern_link)
        P_re+=series_a.sum()
        P_url+=series_b.sum()
        authors.append(list(df['author_id']))
        df=0
    P_re=P_re/total_amount
    P_url=P_url/total_amount
    unqiue_author=len(np.unique(authors))

    all_values=pd.DataFrame()
    for path in all_path:
        df = pd.read_pickle(path)
        all_values=all_values.append(df[['author_id','text','id']])
        df=0
    df=all_values
    all_values=0
    ax=df.groupby('author_id').count().groupby('text').count()['id'].plot(title='Tweets by Same Author')
    ax.set_ylabel('number of authors')
    ax.set_xlabel('counts of tweets')
    ax.figure.savefig('same_author.jpg')

    with open('result.txt', 'w') as f:
        f.write('EDA:'+'\n')
        f.write('- the proportion of tweets that contain a URL is '+str(P_url)+'\n')
        f.write('- the number of unique users is '+str(unqiue_author)+'\n')
        f.write('- the proportion of the data that are retweets '+str(P_re)+'\n')
        f.write('\n')

    def extra_link(text):
        pattern='http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
        return re.findall(pattern,text)

    def check_domain(url_list):
        if url_list:
            try:
                r=requests.head(url_list[0])
                link=r.headers['location']
                domain=urlparse(link).netloc
                return domain
            except:
                try:
                    link=requests.get(url_list[0]).url
                    domain=urlparse(link).netlocA
                    return domain
                except:
                    return ''
        return ''

    count=0
    with open(full_path) as f:
        for line in f:
            load_line=json.loads(line)
            count=count+1
        #    if count<=470315:
        #        continue
            json_record=dict()
            json_record['id']=load_line['id']
            link_list=extra_link(load_line['text'])
            json_record['domain']=check_domain(link_list)
            with open('data/link_data.json','a+', encoding='utf-8') as f:
                json_record = json.dumps(json_record, ensure_ascii=False)
                f.write(json_record + '\n')

    with open(full_path) as f:
        for line in f:
            load_line=json.loads(line)
            json_record=dict()
            json_record['id']=load_line['id']
            link_list=extra_link(load_line['text'])
            json_record['domain']=check_domain(link_list)
            with open('data/link_data.json','a+', encoding='utf-8') as f:
                json_record = json.dumps(json_record, ensure_ascii=False)
                f.write(json_record + '\n')

    misinfo=pd.read_csv('data/iffy.csv')
    domains=misinfo['Domain']

    misinfor_tweets=link_df['domain'].apply(lambda x: x in list(domains))
    proportion_mis=misinfor_tweets.mean()

    fact=['politifact.com', 'factcheck.org', 'washingtonpost.com', 'snopes.com', 'reporterslab.org', 'factcheck.org', 'flackcheck.org', 'mediabiasfactcheck.com', 'npr.org']
    proportion_fact=link_df['domain'].apply(lambda x: x in fact).mean()
    proportion_request=len(link_df[link_df['domain'].apply(lambda x: x !='')])/len(link_df)
    with open('result.txt', 'a') as f:
        f.write('Misinformation:'+'\n')
        f.write('- the proportion of tweets that have a link can be requested is '+str(proportion_request)+'\n')
        f.write('- the proportion of misinformation is '+str(proportion_mis)+'\n')
        f.write('- the proportion of fact check is '+str(proportion_fact)+'\n')
        f.write('\n')


from random import randint, seed
import time
from knapsack_problem import backpack_problem_comprehensive

def time_it(N: int):
    seed(1)
    w = [randint(1, 10) for _ in range(N)]  # waga przedmiotów
    W = 9  # maksymalna waga plecaka
    p = [randint(1, 10) for _ in range(N)]  # wartość przedmiotów

    start_time = time.time()
    print(backpack_problem_comprehensive(w, W, p))
    print(f"For {N} items it took {time.time() - start_time} seconds")

if __name__ == '__main__':
    time_it(24)


#!/usr/bin/python3
import sys

def is_safe(board, row, col, n):
    for i in range(col):
        if board[row][i] == 1:
            return False

    for i, j in zip(range(row, -1, -1), range(col, -1, -1)):
        if board[i][j] == 1:
            return False

    for i, j in zip(range(row, n), range(col, -1, -1)):
        if board[i][j] == 1:
            return False
    return True

def solve_nqueens(n):
    if n < 4:
        print("N must be at least 4")
        sys.exit(1)

    board = [[0 for _ in range(n)] for _ in range(n)]
    solutions = []
    solve_util(board, 0, n, solutions)

    for solution in solutions:
        print(solution)

def solve_util(board, col, n, solutions):
    if col == n:
        solution = []
        for i in range(n):
            for j in range(n):
                if board[i][j] == 1:
                    solution.append([i, j])
        solutions.append(solution)
        return

    for i in range(n):
        if is_safe(board, i, col, n):
            board[i][col] = 1
            solve_util(board, col + 1, n, solutions)
            board[i][col] = 0

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: nqueens N")
        sys.exit(1)
    N = 0
    try:
        N = int(sys.argv[1])
        if N < 4:
            raise TypeError()
        solve_nqueens(N)
    except ValueError:
        print("N must be a number")
        sys.exit(1)
    except TypeError:
        print("N must be at least 4")
        sys.exit(1)


#!/usr/bin/env python3
import re
import argparse
from Bio import SeqIO
import sys

#Args
parser = argparse.ArgumentParser(description='Regex based exact sequence alignment tool\nResult is 1-based')
parser.add_argument('QRY', type=str, nargs = 1,
                    help='fasta of query sequences')
parser.add_argument('REF', type=str, nargs = 1,
                    help='fasta of reference sequences')
parser.add_argument('--version', action='version', version='%(prog)s 0.1')
args = parser.parse_args()

#Prepare fasta
sys.stderr.write("Reading fasta sequences..\n")
qry_dict = SeqIO.to_dict(SeqIO.parse(args.QRY[0], "fasta"))
ref_dict = SeqIO.to_dict(SeqIO.parse(args.REF[0], "fasta"))
ref_dict_reverse = {}
print_buf = ""

#Loop
ref_fa_len = {}
sys.stderr.write("Reversing Ref fasta sequences..\n")
for ref_id in ref_dict:
    ref_dict_reverse[ref_id] = ref_dict[ref_id].seq.reverse_complement()
    ref_fa_len[ref_id] = len(ref_dict[ref_id].seq)

for qry_id in qry_dict.keys():
    result_line = ""
    for ref_id in ref_dict.keys():
        sys.stderr.write("Searching " + qry_id + " in " + ref_id + "..\n")
        result=re.search(qry_dict[qry_id].seq.__str__(), ref_dict[ref_id].seq.__str__(), re.IGNORECASE)
        if(result != None):
            strand = '+'
            result_line= "\t".join([ref_id,str(result.start()+1),
                str(result.end()), qry_id,".",strand])
        else:
            result=re.search(qry_dict[qry_id].seq.__str__(),
                ref_dict_reverse[ref_id].__str__(),re.IGNORECASE)
            if(result != None):
                strand = '-'
                rev_start = ref_fa_len[ref_id] - result.start()
                rev_end = ref_fa_len[ref_id] - result.end() + 1
                result_line= "\t".join([ref_id,str(rev_end),
                    str(rev_start), qry_id,".",strand])
        if(result_line != ""):
            print_buf += result_line + "\n"
print(print_buf, end ='')


from .agent import Agent
from .certificate import lookup
from .principal import Principal
import leb128
import cbor2

def time(agent: Agent, canister_id: str) -> int:
    cert = agent.read_state_raw(canister_id, [["time".encode()]])
    timestamp = lookup(["time".encode()], cert)
    return leb128.u.decode(timestamp)

def subnet_public_key(agent: Agent, canister_id: str, subnet_id: str) -> str:
    path = ["subnet".encode(), Principal.from_str(subnet_id).bytes, "public_key".encode()]
    cert = agent.read_state_raw(canister_id, [path])
    pubkey = lookup(path, cert)
    return pubkey.hex()

def subnet_canister_ranges(agent: Agent, canister_id: str, subnet_id: str) -> list[list[Principal]]:
    path = ["subnet".encode(), Principal.from_str(subnet_id).bytes, "canister_ranges".encode()]
    cert = agent.read_state_raw(canister_id, [path])
    ranges = lookup(path, cert)
    return list(
        map(lambda range:
            list(map(lambda item: Principal(bytes=item), range)),
        cbor2.loads(ranges))
        )

def canister_module_hash(agent: Agent, canister_id: str) -> str:
    path = ["canister".encode(), Principal.from_str(canister_id).bytes, "module_hash".encode()]
    cert = agent.read_state_raw(canister_id, [path])
    module_hash = lookup(path, cert)
    return module_hash.hex()

def canister_controllers(agent: Agent, canister_id: str) -> list[Principal]:
    path = ["canister".encode(), Principal.from_str(canister_id).bytes, "controllers".encode()]
    cert = agent.read_state_raw(canister_id, [path])
    controllers = lookup(path, cert)
    return list(map(lambda item: Principal(bytes=item), cbor2.loads(controllers)))


import torch.utils.data as data
from sklearn.utils import shuffle
from PIL import Image
import os
import os.path

IMG_EXTENSIONS = [
    '.jpg', '.JPG', '.jpeg', '.JPEG',
    '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP',
]

def is_image_file(filename):
    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)

def make_dataset(dir):
    image_paths_A = []
    image_paths_B = []
    path_all = []

    for root, _, fnames in sorted(os.walk(dir)):
        fnames = shuffle(fnames,n_samples=len(fnames))
        for fname in fnames:
            if is_image_file(fname):
                path = os.path.join(root, fname)
                image_paths_A.append(path)
                image_paths_B.append(path.replace("images", "labels").replace("jpg", "png"))

    return image_paths_A, image_paths_B

def default_loader(path):
    return Image.open(path).convert('RGB')

class ImageFolder(data.Dataset):

    def __init__(self, root, transform=None, return_paths=True,
                 loader=default_loader):
        img_paths_A, img_paths_B = make_dataset(root)
        if len(img_paths_A) == 0 or len(img_paths_B) == 0:
            raise(RuntimeError("Found 0 images in subfolders of: " + root + "\n"
                               "Supported image extensions are: " + ",".join(IMG_EXTENSIONS)))

        self.root = root
        self.img_paths_A = img_paths_A
        self.img_paths_B = img_paths_B
        self.transform = transform
        self.return_paths = return_paths
        self.loader = loader

    def __getitem__(self, index):
        path_A = self.img_paths_A[index]
        path_B = self.img_paths_B[index]
        img_A = self.loader(path_A)
        img_B = self.loader(path_B)
        if self.transform is not None:
            img_A = self.transform(img_A)
            img_B = self.transform(img_B)
        if self.return_paths:
            return img_A, img_B, path_A, path_B
        else:
            return img_A, img_B, '', ''

    def __len__(self):
        return len(self.img_paths_A)


from selenium import webdriver
from selenium.webdriver.firefox.service import Service as FirefoxService
from selenium.webdriver.common.by import By
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options

from app.config import db, app
from app.bot.markups import months
from app.models.EconomistIssueModel import EconomistIssue

from datetime import datetime
from dataclasses import dataclass

options = Options()
options.binary_location = r"C:\Program Files\Mozilla Firefox\firefox.exe"
service = FirefoxService(r"C:\Users\sayil\PycharmProjects\cryptovedmabot\geckodriver.exe")

base_url = "https://www.economist.com/weeklyedition/archive?year="

years = [i for i in range(1997, datetime.now().year + 1)]

@dataclass
class Edition:
    editionsUrl:str
    editionsName:str
    editionsDate:str
    editionsImageUrl:str

def parseAllEconomist(base_url: str=base_url):
    print("parseAllEconomist")
    try:
        driver = Firefox(service=service, options=options)
        edition_models = []
        for i in years:
            url = base_url + str(i)
            def parsePage(url: str = url):
                driver.get(url=url)
                editionsUrls = driver.find_elements(by=By.XPATH, value="//a[@class='headline-link']")
                editionsDates = driver.find_elements(by=By.XPATH, value="//time[@class='edition-teaser__subheadline']")
                editionsNames = driver.find_elements(by=By.XPATH, value="//span[@class='edition-teaser__headline']")
                editionsImagesUrl = driver.find_elements(by=By.XPATH, value="//meta[@itemprop='url']")

                editionsUrl = list(map(lambda x: x.get_attribute("href"), editionsUrls))
                editionsName = list(map(lambda x: x.text, editionsUrls))
                editionsDate = list(map(lambda x: x.text, editionsDates))
                editionsImageUrl = list(map(lambda x: x.get_attribute("content"), editionsImagesUrl))

                for i in range(len(editionsUrl)):

                    E = Edition(
                        editionsUrl = editionsUrl[i],
                        editionsDate = editionsDate[i],
                        editionsName = editionsName[i],
                        editionsImageUrl = editionsImageUrl[i]
                    )
                    edition_models.append(E)

            parsePage(url=url)

    except Exception as e:
        print(e)
    finally:
        driver.close()
        for i in edition_models:
            EconomistIssue(
                name=i.editionsName,
                year=i.editionsDate.split()[-1],
                url=i.editionsUrl,
                image_url=i.editionsImageUrl,
                month=months[i.editionsDate.split()[0]]
            )
        print("parseAllEconomistComplete")

new_url = "https://www.economist.com/weeklyedition/archive"

def parseNewEconomist(new_url: str=new_url):
    print("parseNewEconomist")
    try:
        driver = Firefox(service=service, options=options)
        edition_models = []
        def parsePage(url: str = new_url):
            driver.get(url=url)
            editionsUrls = driver.find_elements(by=By.XPATH, value="//a[@class='headline-link']")
            editionsDates = driver.find_elements(by=By.XPATH, value="//time[@class='edition-teaser__subheadline']")
            editionsNames = driver.find_elements(by=By.XPATH, value="//span[@class='edition-teaser__headline']")
            editionsImagesUrl = driver.find_elements(by=By.XPATH, value="//meta[@itemprop='url']")

            editionsUrl = list(map(lambda x: x.get_attribute("href"), editionsUrls))
            editionsName = list(map(lambda x: x.text, editionsUrls))
            editionsDate = list(map(lambda x: x.text, editionsDates))
            editionsImageUrl = list(map(lambda x: x.get_attribute("content"), editionsImagesUrl))
            for i in range(len(editionsUrl)):
                E = Edition(
                    editionsUrl = editionsUrl[i],
                    editionsDate = editionsDate[i],
                    editionsName = editionsName[i],
                    editionsImageUrl = editionsImageUrl[i]
                )
                edition_models.append(E)

        parsePage(url=new_url)

    except Exception as e:
        print(e)
    finally:
        driver.close()
        with app.app_context():
            editions = edition_models
            for edition in editions:
                if EconomistIssue.query.filter_by(url=edition.editionsUrl).first():
                    pass
                else:
                    print("Добавлен", edition.editionsName)
                    EconomistIssue(
                        name=edition.editionsName,
                        year=edition.editionsDate.split()[-1],
                        url=edition.editionsUrl,
                        image_url=edition.editionsImageUrl,
                        month=months[edition.editionsDate.split()[0]]
                    )


import os, sys
f = open("jsondump.json", "r")
o = open("output10k.json", "w")
i = 0
while True:
	line = f.readline()
	if not line:
		break
	i += 1
	if (i % 10000) == 1:
		o.write(line)
	if (i % 100000) == 1:
		sys.stdout.write(".")
		sys.stdout.flush()
o.close()
f.close()


from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from kombu import Connection, Exchange, Queue, Consumer, Producer
import logging

def write_to_rabbitmq():
    try:
        with Connection('amqp://defaultuser:defaultpassword@rabbitmq/') as conn:
            queue = Queue('myqueue', exchange=Exchange('myexchange', type='direct'), routing_key='mykey')
            with conn.channel() as channel:
                producer = Producer(channel)
                message = {
                    'timestamp': str(datetime.now()),
                    'data': 'Hello, RabbitMQ!'
                }
                producer.publish(message, exchange=queue.exchange, routing_key=queue.routing_key)
                logging.info(f"Message sent to RabbitMQ: {message}")
    except Exception as e:
        logging.error(f"Error sending message to RabbitMQ: {e}")

dag = DAG('rabbitmq_write', description='A simple Airflow DAG that writes messages to RabbitMQ queue',
          schedule_interval=None,
          start_date=datetime(2023, 4, 12),
          catchup=False)

write_task = PythonOperator(
    task_id='write_to_rabbitmq',
    python_callable=write_to_rabbitmq,
    dag=dag
)

write_task


import os
import requests
import datetime
import sys
import time
import json
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if root_dir not in sys.path:
    sys.path.append(root_dir)

import grpc
from faiss_grpc import faiss_pb2_grpc, faiss_pb2
from faiss_grpc.faiss_pb2 import UpdateRequest, FaissResponse, Str2FloatMap, FloatArray

def search_vector(vectorType,category):
    channel = grpc.insecure_channel(host_port)
    stub = faiss_pb2_grpc.FaissStub(channel)
    response = stub.search(faiss_pb2.SearchRequest(
        vectorType = vectorType,
        category = category,
        num = 10,
        vector = [FloatArray(val=[1.0 for i in range(64)])],
    ))
    return response

def search_vector_256(vectorType,category):
    channel = grpc.insecure_channel(host_port)
    stub = faiss_pb2_grpc.FaissStub(channel)
    response = stub.search(faiss_pb2.SearchRequest(
        vectorType = vectorType,
        category = category,
        num = 10,
        vector = [FloatArray(val=[1.0 for i in range(256)])],
    ))
    return response

def search_id(vectorType,category,ids):
    channel = grpc.insecure_channel(host_port)
    stub = faiss_pb2_grpc.FaissStub(channel)
    response = stub.search(faiss_pb2.SearchRequest(
        vectorType = vectorType,
        category = category,
        num = 10,
        itemId = ids
    ))
    return response

def get_version(response):
    return response.version

def get_neighbors(response):
    neighbors = dict()
    for k, v in response.similarItems.items():
        neighbors[k] = dict(v.innerMap)
    return neighbors

def get_id(response):
    neighbors = dict()
    for k, v in response.similarItems.items():
        neighbors[0] = dict(v.innerMap)
    return neighbors[0].keys()

def do_monitor(category_list,vectorType, monitorType):

    for category in category_list:

        _Normal = 0
        _ResAbnormal = 1
        _VersionAbnormal = 2
        _NumAbnormal = 3

        res = None
        metric = None
        if monitorType == "monitors_id":
            resp = search_vector(vectorType,category)
            res = search_id(vectorType,category,get_id(resp))
            metric = "faiss_" + vectorType + "/" + category + "_id_monitor"
        elif monitorType == "monitors_vector":
            res = search_vector(vectorType,category)
            metric = "faiss_" + vectorType + "/" + category + "_vector_monitor"
        elif monitorType == "monitors_vector_256":
            res = search_vector_256(vectorType,category)
            metric = "faiss_" + vectorType + "/" + category + "_vector_monitor"
            #print(res)

        version=get_version(res)
        #print(version)
        handler=get_neighbors(res)
        #print(handler)

        value = _Normal
        ts = int(time.time())
        step = 300
        payload = []

        #print(len(list((list(handler.values()))[0])))

        if handler == None:
            value = _ResAbnormal

        elif ("card" not in metric and len(list((list(handler.values()))[0])) < 100) or ("card" in metric and len(list((list(handler.values()))[0])) < 20):
            value = _NumAbnormal

        elif version != None and "swing" in metric:

            version = version[8:14]
            print(version)
            c_time = (datetime.datetime.now() + datetime.timedelta(days=-3)).strftime('%Y%m%d')[2:]

            if version < c_time:
                value = _VersionAbnormal
        else:
            pass

        mes ={
            "endpoint": "newfaiss",
            "metric": metric,
            "timestamp": ts,
            "step": step,
            "value": value,
            "counterType": "GAUGE",
            "tags": "result"
          }
        payload.append(mes)
        print(payload)
        print(value)
        requests.post("http://127.0.0.1:1988/v1/push", data=json.dumps(payload))
        r = requests.post("http://127.0.0.1:1988/v1/push", data=json.dumps(payload))
        print(r.text)

if __name__ == '__main__':

    host_port='faiss-new.internal.mxplay.com:8080'
    #host_port='ec2-13-235-99-239.ap-south-1.compute.amazonaws.com:50051'

    tab_id = ['08f8fce450d1ecf00efa820f611cf57b','5f15a7356cf9dfceeb355fc49dd2be8a','4e82d34404a477419f811cd303e216e7','4a0eba2ea3b96a2ff2a72f60a0f1693b','6256019abbad655653afffa3056345a1','a78e8100f00962aa48326fd00fed5908','7694f56f59238654b3a6303885f9166f','87c3ddc974dcf12294e9412bec44b097','feacc8bb9a44e3c86e2236381f6baaf3']
    tab_id2 = ['a78e8100f00962aa48326fd00fed5908','08f8fce450d1ecf00efa820f611cf57b','feacc8bb9a44e3c86e2236381f6baaf3']
    tab_id3 = ['08f8fce450d1ecf00efa820f611cf57b','4e82d34404a477419f811cd303e216e7','6256019abbad655653afffa3056345a1','a78e8100f00962aa48326fd00fed5908','7694f56f59238654b3a6303885f9166f','87c3ddc974dcf12294e9412bec44b097','feacc8bb9a44e3c86e2236381f6baaf3']
    category_type1 = ['music','tv_show','short_video','movie']
    category_type2 = ['music','tv_show','short_video','movie','episode','season','singer','publisher','playlist']

    do_monitor(tab_id,'deepwalk/online/card','monitors_vector')
    do_monitor(tab_id,'deepwalk/all_mix/card','monitors_vector')
    do_monitor(tab_id2,'deepwalk/online_mix/card','monitors_vector')
    do_monitor(tab_id3,'deepwalk/dnn_vector/card','monitors_vector_256') #except news/web series
    do_monitor(category_type1,'research/deepwalk/online/swing','monitors_vector')
    do_monitor(category_type2,'deepwalk/online','monitors_id')
    do_monitor(category_type1,'research/deepwalk/online/swing','monitors_id')
    do_monitor(category_type2,'deepwalk/online','monitors_vector')


from .models import Empinsert
from django.core.mail import send_mail
from django.conf import settings

from django.shortcuts import render
from flask import Flask, request

# Create your views here.
def home(request):
    return render(request, 'home1.html')

#  return HttpResponse('hey1!!!!!!!!!!!!!!!!!')

def add_form(request):
    print("sucess")
    name = request.POST["name"]
    mail = request.POST["mail"]
    number = request.POST["number"]
    age = request.POST["age"]
    state = request.POST["state"]
    msg = request.POST["msg"]
    emp_insert = Empinsert(name=name, mail=mail, number=number, age=age, state=state, msg=msg)

    subject = 'Thank you for registration. we will text you soon'
    message = 'welcome to Modern driving training school'
    from_email = settings.EMAIL_HOST_USER
    to_list = [mail, settings.EMAIL_HOST_USER]
    send_mail(subject, message, from_email, to_list, fail_silently=False)
    emp_insert.save()
    return render(request, 'home2.html')


from __future__ import unicode_literals

# Java imports.
from org.semanticweb.owlapi.util import SimpleIRIMapper
from org.semanticweb.owlapi.apibinding import OWLManager

# Stores all custom mappings of ontology IRIs to document IRIs.
_IRImappings = {}

# Keeps track of all OOMs created by the OOM manager.  This implementation is
# somewhat inefficient, because OOMs will never be deallocated/garbage
# collected due to this list always keeping a reference to them.  In practice,
# this is unlikely to matter because few OOMs will be instantiated during a
# typical OntoPilot run.  Even when running the unit tests, which create a
# *lot* of OOMs, it hardly made a difference in overall memory consumption:
# e.g., for one pair of runs, I observed a peak of 722 MB without storing
# references and 733 MB with reference storing.  However, if needed, this could
# be optimized to improve memory management.
_ooms_list = []

def lookupDocumentIRI(oom, ontologyIRI):
    """
    Given an OOM, checks all IRI mappers registered with the OOM until a
    mapping for the ontology IRI is found.  If no mapping is found, returns
    None.  This is functionality that appears to be missing in the
    OWLOntologyManager public interface.

    oom: An OWLOntologyManager.
    ontologyIRI: An OWL API IRI object.
    """
    for mapper in oom.getIRIMappers():
        res = mapper.getDocumentIRI(ontologyIRI)
        if res is not None:
            return res

    return None

def addNewIRIMapping(ontologyIRI, documentIRI):
    """
    Adds a new mapping of an ontology IRI to a document IRI.  All previously
    created OOMs will be updated with the new mapping, and the new mapping will
    automatically be applied to any new OOMs that are created after the mapping
    is added.

    ontologyIRI: An OWL API IRI object.
    documentIRI: An OWL API IRI object.
    """
    if ontologyIRI.equals(documentIRI):
        return

    if ontologyIRI not in _IRImappings:
        _IRImappings[ontologyIRI] = documentIRI

        for oom in _ooms_list:
            if lookupDocumentIRI(oom, ontologyIRI) is None:
                oom.getIRIMappers().add(
                    SimpleIRIMapper(ontologyIRI, documentIRI)
                )

    elif not(_IRImappings[ontologyIRI].equals(documentIRI)):
        raise RuntimeError(
            'Could not create the mapping of <{0}> to <{1}>, because a '
            'conflicting mapping of <{0}> to <{2}> already exists.'.format(
                    ontologyIRI.toString(), documentIRI.toString(),
                    _IRImappings[ontologyIRI].toString()
            )
        )

def getNewOWLOntologyManager():
    """
    Creates and returns a new OWLOntologyManager.
    """
    oom = OWLManager.createOWLOntologyManager()
    _ooms_list.append(oom)

    # Add any missing IRI mappings.
    for ontologyIRI in _IRImappings:
        if lookupDocumentIRI(oom, ontologyIRI) is None:
            oom.getIRIMappers().add(
                SimpleIRIMapper(ontologyIRI, _IRImappings[ontologyIRI])
            )

    return oom


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
import sys
import gzip
import json
import logging
import os
import shutil
import subprocess
import tempfile
import time
import traceback
import six

if sys.version_info.major == 3:
  JSON_FILE_MODE = 'w+'
else:
  JSON_FILE_MODE = 'w+b'

try:
  StringTypes = six.string_types # pylint: disable=invalid-name
except NameError:
  StringTypes = str

_TRACING_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                            os.path.pardir, os.path.pardir)
_TRACE2HTML_PATH = os.path.join(_TRACING_DIR, 'bin', 'trace2html')

MIB = 1024 * 1024

class TraceDataPart(object):
  """Trace data can come from a variety of tracing agents.

  Data from each agent is collected into a trace "part" and accessed by the
  following fixed field names.
  """
  def __init__(self, raw_field_name):
    self._raw_field_name = raw_field_name

  def __repr__(self):
    return 'TraceDataPart("%s")' % self._raw_field_name

  @property
  def raw_field_name(self):
    return self._raw_field_name

  def __eq__(self, other):
    return self.raw_field_name == other.raw_field_name

  def __hash__(self):
    return hash(self.raw_field_name)

ANDROID_PROCESS_DATA_PART = TraceDataPart('androidProcessDump')
ATRACE_PART = TraceDataPart('systemTraceEvents')
ATRACE_PROCESS_DUMP_PART = TraceDataPart('atraceProcessDump')
CHROME_TRACE_PART = TraceDataPart('traceEvents')
CPU_TRACE_DATA = TraceDataPart('cpuSnapshots')
TELEMETRY_PART = TraceDataPart('telemetry')
WALT_TRACE_PART = TraceDataPart('waltTraceEvents')
CGROUP_TRACE_PART = TraceDataPart('cgroupDump')

ALL_TRACE_PARTS = {ANDROID_PROCESS_DATA_PART,
                   ATRACE_PART,
                   ATRACE_PROCESS_DUMP_PART,
                   CHROME_TRACE_PART,
                   CPU_TRACE_DATA,
                   TELEMETRY_PART}

class _TraceData(object):
  """Provides read access to traces collected from multiple tracing agents.

  Instances are created by calling the AsData() method on a TraceDataWriter.
  """
  def __init__(self, raw_data):
    self._raw_data = raw_data

  def HasTracesFor(self, part):
    return bool(self.GetTracesFor(part))

  def GetTracesFor(self, part):
    """Return the list of traces for |part| in string or dictionary forms."""
    if not isinstance(part, TraceDataPart):
      raise TypeError('part must be a TraceDataPart instance')
    return self._raw_data.get(part.raw_field_name, [])

  def GetTraceFor(self, part):
    traces = self.GetTracesFor(part)
    assert len(traces) == 1
    return traces[0]

_TraceItem = collections.namedtuple(
    '_TraceItem', ['part_name', 'handle'])

class TraceDataException(Exception):
  """Exception raised by TraceDataBuilder via RecordTraceDataException()."""

class TraceDataBuilder(object):
  """TraceDataBuilder helps build up a trace from multiple trace agents.

  Note: the collected trace data is maintained in a set of temporary files to
  be later processed e.g. by the Serialize() method. To ensure proper clean up
  of such files clients must call the CleanUpTraceData() method or, even easier,
  use the context manager API, e.g.:

      with trace_data.TraceDataBuilder() as builder:
        builder.AddTraceFor(trace_part, data)
        builder.Serialize(output_file)
  """
  def __init__(self):
    self._traces = []
    self._frozen = False
    self._temp_dir = tempfile.mkdtemp()
    self._exceptions = []

  def __enter__(self):
    return self

  def __exit__(self, *args):
    self.CleanUpTraceData()

  def OpenTraceHandleFor(self, part, suffix, mode=None):
    """Open a file handle for writing trace data into it.

    Args:
      part: A TraceDataPart instance.
      suffix: A string used as file extension and identifier for the format
        of the trace contents, e.g. '.json'. Can also append '.gz' to
        indicate gzipped content, e.g. '.json.gz'.
    """
    if not isinstance(part, TraceDataPart):
      raise TypeError('part must be a TraceDataPart instance')
    if self._frozen:
      raise RuntimeError('trace data builder is no longer open for writing')
    if mode:
      trace = _TraceItem(
          part_name=part.raw_field_name,
          handle=tempfile.NamedTemporaryFile(
              mode=mode, delete=False, dir=self._temp_dir, suffix=suffix))
    else:
      trace = _TraceItem(
          part_name=part.raw_field_name,
          handle=tempfile.NamedTemporaryFile(
              delete=False, dir=self._temp_dir, suffix=suffix))
    self._traces.append(trace)
    return trace.handle

  def AddTraceFileFor(self, part, trace_file):
    """Move a file with trace data into this builder.

    This is useful for situations where a client might want to start collecting
    trace data into a file, even before the TraceDataBuilder itself is created.

    Args:
      part: A TraceDataPart instance.
      trace_file: A path to a file containing trace data. Note: for efficiency
        the file is moved rather than copied into the builder. Therefore the
        source file will no longer exist after calling this method; and the
        lifetime of the trace data will thereafter be managed by this builder.
    """
    _, suffix = os.path.splitext(trace_file)
    with self.OpenTraceHandleFor(part, suffix) as handle:
      pass
    if os.name == 'nt':
      # On windows os.rename won't overwrite, so the destination path needs to
      # be removed first.
      os.remove(handle.name)
    os.rename(trace_file, handle.name)

  def AddTraceFor(self, part, data, allow_unstructured=False):
    """Record new trace data into this builder.

    Args:
      part: A TraceDataPart instance.
      data: The trace data to write: a json-serializable dict, or unstructured
        text data as a string.
      allow_unstructured: This must be set to True to allow passing
        unstructured text data as input. Note: the use of this flag is
        discouraged and only exists to support legacy clients; new tracing
        agents should all produce structured trace data (e.g. proto or json).
    """
    if isinstance(data, StringTypes):
      if not allow_unstructured:
        raise ValueError('must pass allow_unstructured=True for text data')
      do_write = lambda d, f: f.write(d)
      suffix = '.txt'  # Used for atrace and systrace data.
    elif isinstance(data, dict):
      do_write = json.dump
      suffix = '.json'
    else:
      raise TypeError('invalid trace data type')
    with self.OpenTraceHandleFor(part, suffix, JSON_FILE_MODE) as handle:
      do_write(data, handle)

  def Freeze(self):
    """Do not allow writing any more data into this builder."""
    self._frozen = True
    return self

  def CleanUpTraceData(self):
    """Clean up resources used by the data builder.

    Will also re-raise any exceptions previously added by
    RecordTraceCollectionException().
    """
    if self._traces is None:
      return  # Already cleaned up.
    self.Freeze()
    for trace in self._traces:
      # Make sure all trace handles are closed. It's fine if we close some
      # of them multiple times.
      trace.handle.close()
    shutil.rmtree(self._temp_dir)
    self._temp_dir = None
    self._traces = None

    if self._exceptions:
      raise TraceDataException(
          'Exceptions raised during trace data collection:\n' +
          '\n'.join(self._exceptions))

  def Serialize(self, file_path, trace_title=None):
    """Serialize the trace data to a file in HTML format."""
    self.Freeze()
    assert self._traces, 'trace data has already been cleaned up'

    trace_files = [trace.handle.name for trace in self._traces]
    SerializeAsHtml(trace_files, file_path, trace_title)

  def AsData(self):
    """Allow in-memory access to read the collected JSON trace data.

    This method is only provided for writing tests which require read access
    to the collected trace data (e.g. for tracing agents to test they correctly
    write data), and to support legacy TBMv1 metric computation. Only traces
    in JSON format are supported.

    Be careful: this may require a lot of memory if the traces to process are
    very large. This has lead in the past to OOM errors (e.g. crbug/672097).

    TODO(crbug/928278): Ideally, this method should be removed when it can be
    entirely replaced by calls to an external trace processor.
    """
    self.Freeze()
    assert self._traces, 'trace data has already been cleaned up'

    raw_data = {}
    for trace in self._traces:
      is_compressed_json = trace.handle.name.endswith('.json.gz')
      is_json = trace.handle.name.endswith('.json') or is_compressed_json
      if is_json:
        traces_for_part = raw_data.setdefault(trace.part_name, [])
        opener = gzip.open if is_compressed_json else open
        with opener(trace.handle.name, 'rb') as f:
          traces_for_part.append(json.load(f))
      else:
        logging.info('Skipping over non-json trace: %s', trace.handle.name)
    return _TraceData(raw_data)

  def IterTraceParts(self):
    """Iterates over trace parts.

    Return value: iterator over pairs (part_name, file_path).
    """
    for trace in self._traces:
      yield trace.part_name, trace.handle.name

  def RecordTraceDataException(self):
    """Records the most recent exception to be re-raised during cleanup.

    Exceptions raised during trace data collection can be stored temporarily
    in the builder. They will be re-raised when the builder is cleaned up later.
    This way, any collected trace data can still be retained before the
    benchmark is aborted.

    This method is intended to be called from within an "except" handler, e.g.:
      try:
        # Collect trace data.
      except Exception: # pylint: disable=broad-except
        builder.RecordTraceDataException()
    """
    self._exceptions.append(traceback.format_exc())

def CreateTestTrace(number=1):
  """Convenient helper method to create trace data objects for testing.

  Objects are created via the usual trace data writing route, so clients are
  also responsible for cleaning up trace data themselves.

  Clients are meant to treat these test traces as opaque. No guarantees are
  made about their contents, which they shouldn't try to read.
  """
  builder = TraceDataBuilder()
  builder.AddTraceFor(CHROME_TRACE_PART, {'traceEvents': [{'test': number}]})
  return builder.Freeze()

def CreateFromRawChromeEvents(events):
  """Convenient helper to create trace data objects from raw Chrome events.

  This bypasses trace data writing, going directly to the in-memory json trace
  representation, so there is no need for trace file cleanup.

  This is used only for testing legacy clients that still read trace data.
  """
  assert isinstance(events, list)
  return _TraceData({
      CHROME_TRACE_PART.raw_field_name: [{'traceEvents': events}]})

def SerializeAsHtml(trace_files, html_file, trace_title=None):
  """Serialize a set of traces to a single file in HTML format.

  Args:
    trace_files: a list of file names, each containing a trace from
        one of the tracing agents.
    html_file: a name of the output file.
    trace_title: optional. A title for the resulting trace.
  """
  if not trace_files:
    raise ValueError('trace files list is empty')

  input_size = sum(os.path.getsize(trace_file) for trace_file in trace_files)

  cmd = [sys.executable]
  cmd.append(_TRACE2HTML_PATH)
  cmd.extend(trace_files)
  cmd.extend(['--output', html_file])
  if trace_title is not None:
    cmd.extend(['--title', trace_title])

  start_time = time.time()
  subprocess.check_output(cmd)
  elapsed_time = time.time() - start_time
  logging.info('trace2html processed %.01f MiB of trace data in %.02f seconds.',
               1.0 * input_size / MIB, elapsed_time)


import json
import pickle
import os
from models.product import Product
from models.user_data import UserData
from .scrap import Scrap

PRODUCT_FILE = "Project\database\product.bin"
USER_FILE = "Project\database\\user.bin"

class Database():
    def __init__(self):
        # Initialize the Scrapper instance
        self.bot = Scrap(teardown=True)

    # Helper function for getting and setting data to the database for both Product and User

    # Fetches data from the database referred to by the file name and returns it as a list of objects
    def getData(self,file):
        data = []
        try:
            with open(file,'rb') as f:
                while True:
                    try:
                        data.append(pickle.load(f))
                    except EOFError:
                        break
                f.close()
        except FileNotFoundError:
            pass
        return data

    # Stores data in the database referred to by the file name and returns True if the insertion was successful
    def storeData(self,file,data):
        try:
            with open(file,'ab') as f:
                for item in data:
                    pickle.dump(item,f,protocol=pickle.HIGHEST_PROTOCOL)
            return True
        except FileNotFoundError:
            return False

    # Functions related to Product

    # Add the specified product to the product table
    def addProduct(self,newProduct : Product):
        data = self.getData(PRODUCT_FILE)
        if data == []:
            newProduct.productId = 0
        else:
            newProduct.productId = data[-1].productId +1
        self.storeData(PRODUCT_FILE,[newProduct])
        return newProduct

    # Get a list of all the tracked products for the specified user
    def getUserProducts(self,userId):
        data = self.getData(PRODUCT_FILE)
        result = []
        for product in data:
            if product.userId == userId:

                result.append(product)

        return result

    # Update the details of the tracked product with new information fetched from the URL
    def updatePrices(self,userId):
        data = self.getData(PRODUCT_FILE)
        for index,product in enumerate(data):
            if product.userId == userId:
                tempProduct = self.bot.getProductDetails(product.url)
                product.currentPrice = tempProduct.currentPrice
                product.currentDate = tempProduct.currentDate
                product.rating = tempProduct.rating
                del data[index]
                data.append(product)
                os.remove(PRODUCT_FILE)
                self.storeData(PRODUCT_FILE,data)
        return "Product Updated"

    # Remove the specified product from the product table
    def removeProduct(self,productId : Product,userId : UserData):
        data = self.getData(PRODUCT_FILE)
        for index,product in enumerate(data):
            if product.productId == productId and product.userId == userId:
                del data[index]
                os.remove(PRODUCT_FILE)
                self.storeData(PRODUCT_FILE,data)
                return "Product Deleted"
        return "Product not Found"

    # Get details of the product from the given URL
    def getProductDetails(self,url):
        return self.bot.getProductDetails(url)

    # Functions related to User

    # Validates the user credentials and returns the UserData object if successful else returns None
    def login(self,username,password):
        data = self.getData(USER_FILE)

        for user in data:
            if(user.userName == username and user.password == password):
                return user
        return None

    # Add the UserData object to the user table if the username is unique
    def registerUser(self, newUser : UserData):
        data = self.getData(USER_FILE)

        for user in data:
            if(newUser.userName == user.userName):
                return "User already exists"

        if data == []:
            newUser.userId = 0
        else:
            newUser.userId = data[-1].userId + 1
        return self.storeData(USER_FILE,[newUser])

    # Delete a UserData object from the user table
    def deleteUser(self, userId):
        data = self.getData(USER_FILE)
        for index,user in enumerate(data):
            if(user.userId == userId):
                del data[index]
                os.remove(USER_FILE)
                self.storeData(USER_FILE,data)
                return "User deleted"
        return "User not found"


"""
START -> LIST_OF_TOP_LEVEL_EXPRESSIONS

LIST_OF_TOP_LEVEL_EXPRESSIONS -> TOP_LEVEL_EXPRESSION
    | LIST_OF_TOP_LEVEL_EXPRESSIONS TOP_LEVEL_EXPRESSION

TOP_LEVEL_EXPRESSION -> TOP_LEVEL_TYPE_DEFINITION
    | TOP_LEVEL_EXPRESSION_DEFINITION

TOP_LEVEL_TYPE_DEFINITION -> EXPRESSION_OPENNING TYPE_DEFINITION_MARK EXPRESSION_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING

TYPE_DESIGNATOR -> EXPRESSION_NAME
                 | BASIC_TYPE

TOP_LEVEL_EXPRESSION_DEFINITION -> EXPRESSION_OPENNING LIST_OF_EXPRESSIONS EXPRESSION_CLOSING

LIST_OF_EXPRESSIONS -> EXPRESSION
                     | LIST_OF_EXPRESSIONS EXPRESSION

EXPRESSION -> EXPRESSION_OPENNING LIST_OF_EXPRESSIONS EXPRESSION_CLOSING
            | EXPRESSION_OPENNING TYPE_DEFINITION_MARK EXPRESSION_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING
            | EXPRESSION_NAME
            | TERMINAL_NAME
"""

from typing import List, Any, Optional
import ply.yacc as yacc
from das.metta_lex import MettaLex
from das.exceptions import MettaSyntaxError, UndefinedSymbolError
from das.expression_hasher import ExpressionHasher
from das.expression import Expression
from das.base_yacc import BaseYacc

class MettaYacc(BaseYacc):

    ### Parser rules ###

    def p_START(self, p):
        """START : LIST_OF_TOP_LEVEL_EXPRESSIONS EOF
                 | EOF
                 |"""
        self._revisit_pending_symbols()
        missing_symbols = []
        missing_symbols.extend([name for (name, expression) in self.pending_terminal_names])
        missing_symbols.extend([name for (name, expression) in self.pending_expression_names])
        missing_symbols.extend([type_designator for ((name, type_designator), expression) in self.pending_named_types])
        if missing_symbols:
            raise UndefinedSymbolError(list(set(missing_symbols)))
        assert not self.pending_expressions
        p[0] = 'SUCCESS'

    def p_LIST_OF_TOP_LEVEL_EXPRESSIONS_base(self, p):
        """LIST_OF_TOP_LEVEL_EXPRESSIONS : TOP_LEVEL_EXPRESSION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = [p[1]]

    def p_LIST_OF_TOP_LEVEL_EXPRESSIONS_recursion(self, p):
        """LIST_OF_TOP_LEVEL_EXPRESSIONS :  LIST_OF_TOP_LEVEL_EXPRESSIONS TOP_LEVEL_EXPRESSION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = [*p[1], p[2]]

    def p_TOP_LEVEL_EXPRESSION_type(self, p):
        """TOP_LEVEL_EXPRESSION : TOP_LEVEL_TYPE_DEFINITION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = p[1]

    def p_TOP_LEVEL_EXPRESSION_expression(self, p):
        """TOP_LEVEL_EXPRESSION : TOP_LEVEL_EXPRESSION_DEFINITION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = p[1]

    def p_TOP_LEVEL_TYPE_DEFINITION(self, p):
        """TOP_LEVEL_TYPE_DEFINITION : EXPRESSION_OPENNING TYPE_DEFINITION_MARK EXPRESSION_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING
                                     | EXPRESSION_OPENNING TYPE_DEFINITION_MARK TERMINAL_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING"""
        if self.check_mode or not self.action_broker:
            return
        assert self.typedef_mark == p[2]
        name = p[3]
        type_designator = p[4]
        expression = self._typedef(name, type_designator)
        expression.toplevel = True
        self.action_broker.new_top_level_typedef_expression(expression)
        p[0] = expression

    def p_TYPE_DESIGNATOR_expression(self, p):
        """TYPE_DESIGNATOR : EXPRESSION_NAME"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = p[1]

    def p_TYPE_DESIGNATOR_basic(self, p):
        """TYPE_DESIGNATOR : BASIC_TYPE"""
        if self.check_mode or not self.action_broker:
            return
        name = p[1]
        named_type_hash = self._get_named_type_hash(name)
        self.parent_type[named_type_hash] = named_type_hash
        p[0] = name

    def p_TOP_LEVEL_EXPRESSION_DEFINITION(self, p):
        """TOP_LEVEL_EXPRESSION_DEFINITION : EXPRESSION_OPENNING LIST_OF_EXPRESSIONS EXPRESSION_CLOSING"""
        if self.check_mode or not self.action_broker:
            return
        sub_expressions = p[2]
        expression = self._nested_expression(sub_expressions)
        expression.toplevel = True
        self.action_broker.new_top_level_expression(expression)
        p[0] = expression

    def p_LIST_OF_EXPRESSIONS_base(self, p):
        """LIST_OF_EXPRESSIONS : EXPRESSION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = [p[1]]

    def p_LIST_OF_EXPRESSIONS_recursion(self, p):
        """LIST_OF_EXPRESSIONS : LIST_OF_EXPRESSIONS EXPRESSION"""
        if self.check_mode or not self.action_broker:
            return
        p[0] = [*p[1], p[2]]

    def p_EXPRESSION_sequence(self, p):
        """EXPRESSION : EXPRESSION_OPENNING LIST_OF_EXPRESSIONS EXPRESSION_CLOSING"""
        if self.check_mode or not self.action_broker:
            return
        sub_expressions = p[2]
        expression = self._nested_expression(sub_expressions)
        self.action_broker.new_expression(expression)
        p[0] = expression

    def p_EXPRESSION_type(self, p):
        """EXPRESSION : EXPRESSION_OPENNING TYPE_DEFINITION_MARK EXPRESSION_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING
                      | EXPRESSION_OPENNING TYPE_DEFINITION_MARK TERMINAL_NAME TYPE_DESIGNATOR EXPRESSION_CLOSING"""
        if self.check_mode or not self.action_broker:
            return
        assert self.typedef_mark == p[2]
        name = p[3]
        type_designator = p[4]
        expression = self._typedef(name, type_designator)
        error = f"Error in line {self.lexer.lineno} " + \
                f"Invalid nested type definition: {name}. " + \
                f"Current token: {p}"
        raise MettaSyntaxError(error)

    def p_EXPRESSION_symbol(self, p):
        """EXPRESSION : EXPRESSION_NAME"""
        if self.check_mode or not self.action_broker:
            return
        expression_name = p[1]
        expression = self._new_symbol(expression_name)
        p[0] = expression

    def p_EXPRESSION_terminal(self, p):
        """EXPRESSION : TERMINAL_NAME"""
        if self.check_mode or not self.action_broker:
            return
        terminal_name = p[1]
        expression = self._new_terminal(terminal_name)
        self.action_broker.new_terminal(expression)
        p[0] = expression

    def p_error(self, p):
        error = f"Syntax error in line {self.lexer.lineno} " + \
                f"Current token: {p}"
        raise MettaSyntaxError(error)

    ### End of parser rules ###

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.lex_wrap = MettaLex()
        super().setup()
        self.parser = yacc.yacc(module=self)


import logging
import math
import pathlib
import sys

import fitz  # PyMuPDF

from .conversion.parsing import (
    check_rm_file_version,
    parse_rm_file,
    rescale_parsed_data,
    get_ann_max_bound,
)
from .conversion.text import (
    check_if_text_extractable,
    extract_groups_from_pdf_ann_hl,
    extract_groups_from_smart_hl,
    prepare_md_from_hl_groups,
)
from .conversion.ocrmypdf import (
    is_executable_available,
    run_ocr,
)
from .conversion.drawing import (
    draw_annotations_on_pdf,
    add_smart_highlight_annotations,
)
from .utils import (
    is_document,
    get_document_filetype,
    get_visible_name,
    get_ui_path,
    get_pages_data,
    list_ann_rm_files,
    list_hl_json_files,
    load_json_file,
    prepare_subdir,
    rescale_given_device_aspect_ratio,
    RM_WIDTH,
    RM_HEIGHT,
)

# TODO: add support to `.textconversion/*.json` files, that's an easy way to
# start offering some support to handwriting conversion...
#
# See also:
# - https://github.com/lucasrla/remarks/issues/13
# - https://github.com/lucasrla/remarks/issues/11

def run_remarks(
    input_dir, output_dir, file_name=None, file_uuid=None, file_path=None, **kwargs
):
    num_docs = sum(1 for _ in pathlib.Path(f"{input_dir}/").glob("*.metadata"))

    if num_docs == 0:
        logging.warning(
            f'No .metadata files found in "{input_dir}". Are you sure you\'re running remarks on a valid xochitl-like directory? See: https://github.com/lucasrla/remarks#1-copy-remarkables-raw-document-files-to-your-computer'
        )
        sys.exit(1)

    logging.info(
        f'\nFound {num_docs} documents in "{input_dir}", will process them now',
    )

    for metadata_path in pathlib.Path(f"{input_dir}/").glob("*.metadata"):
        if file_uuid is not None and metadata_path.stem != file_uuid:
            continue

        if not is_document(metadata_path):
            continue

        doc_type = get_document_filetype(metadata_path)
        # Both "Quick Sheets" and "Notebooks" have doc_type="notebook"
        supported_types = ["pdf", "epub", "notebook"]

        doc_name = get_visible_name(metadata_path)

        if (file_name and (file_name not in doc_name)) or not doc_name:
            continue

        if doc_type in supported_types:
            logging.info(f'\nFile: "{doc_name}.{doc_type}" ({metadata_path.stem})')

            in_device_dir = get_ui_path(metadata_path)
            out_path = pathlib.Path(f"{output_dir}/{in_device_dir}/{doc_name}/")
            # print("out_path:", out_path)

            if file_path is not None and file_path not in str(in_device_dir):
                continue

            process_document(metadata_path, out_path, doc_type, **kwargs)
        else:
            logging.info(
                f'\nFile skipped: "{doc_name}" ({metadata_path.stem}) due to unsupported filetype: {doc_type}. remarks only supports: {", ".join(supported_types)}'
            )

    logging.info(
        f'\nDone processing "{input_dir}"',
    )

# TODO: review args
def process_document(
    metadata_path,
    out_path,
    doc_type,
    per_page_targets=None,
    ann_type=None,
    combined_pdf=False,
    modified_pdf=False,
    combined_md=False,
    assume_malformed_pdfs=False,
    avoid_ocr=False,
    md_hl_format="whole_block",
    md_page_offset=0,
    md_header_format="atx",
):
    pages_list, pages_map = get_pages_data(metadata_path)

    if len(pages_list) == 0:
        return

    pages_magnitude = math.floor(math.log10(len(pages_list))) + 1

    ann_rm_files = list_ann_rm_files(metadata_path)  # scribbles
    # print("ann_rm_files", ann_rm_files)
    hl_json_files = list_hl_json_files(metadata_path)  # highlights
    # print("hl_json_files", hl_json_files)

    if ann_type == "scribbles" and len(ann_rm_files) == 0:
        logging.info(
            "- You asked for scribbles, but we couldn't find any of those on this document. Will skip this one"
        )
        return

    if ann_type == "highlights" and len(hl_json_files) == 0 and len(ann_rm_files) == 0:
        logging.info(
            "- You asked for highlights, but we couldn't find anything highlighted on this document. Will skip this one"
        )
        return

    if len(hl_json_files) == 0 and len(ann_rm_files) == 0:
        logging.info(
            "- Found nothing annotated on this document (no scribbles, no highlights). Will skip this one"
        )
        return

    if combined_md:
        combined_md_strs = []

    if modified_pdf:
        mod_pdf = fitz.open()
        pages_order = []

    # PyMuPDF's A4 default is width=595, height=842
    # - https://pymupdf.readthedocs.io/en/latest/document.html#Document.new_page
    # The 0.42 below is just me eye-balling PyMuPDF's defaults:
    # 1404*0.42 ~= 590 and 1872*0.42 ~= 786
    #
    # reMarkable's desktop app exports notebooks to PDF with 445 x 594, in
    # terms of scale it is 445/1404 = ~0.316
    note_page_dims = (RM_WIDTH * 0.42, RM_HEIGHT * 0.42)

    # Open the original PDF source document
    if doc_type in ["pdf", "epub"]:
        f = metadata_path.with_name(f"{metadata_path.stem}.pdf")
        pdf_src = fitz.open(f)

    # Thanks to @apoorvkh
    # - https://github.com/lucasrla/remarks/issues/11#issuecomment-1287175782
    # - https://github.com/apoorvkh/remarks/blob/64dd3b586b96195b00e727fc1f1e537b90d841dc/remarks/remarks.py#L16-L38
    elif doc_type == "notebook":
        # Open an empty PDF to be treated as if it were the original document
        pdf_src = fitz.open()

        # Create its first page according to reMarkable device's aspect ratio
        pdf_src.new_page(width=note_page_dims[0], height=note_page_dims[1])

        # The mapped value of any first page is always `0`, any other note
        # pages are always `-1` (including the ones inserted into PDF/EPUBs)
        pages_map = [0] + [-1 for _ in range(len(pages_list) - 1)]

    # print("pages_list, pages_map", pages_list, pages_map)

    # If this is part of a notebook, use those arbitrary dimensions
    if doc_type == "notebook":
        blank_page_dims = note_page_dims
    # If this is part of an actual PDF/EPUB, use dimensions of its first page
    # (index=0)
    else:
        blank_page_dims = (pdf_src[0].rect.width, pdf_src[0].rect.height)

    # For each note page, add a blank page to the original document
    for i, page_idx in enumerate(pages_map):
        if page_idx == -1:
            pdf_src.new_page(
                width=blank_page_dims[0],
                height=blank_page_dims[1],
                pno=i,
            )

    pages_to_process = set(
        [f.stem for f in ann_rm_files] + [f.stem for f in hl_json_files]
    )

    for page_uuid in pages_to_process:
        page_idx = pages_list.index(f"{page_uuid}")
        # print("page_uuid:", page_uuid)
        # print("page_idx", page_idx)

        ann_rm_file = None
        hl_json_file = None

        has_ann = False
        has_smart_hl = False
        has_ann_hl = False

        for f in ann_rm_files:
            if page_uuid == f.stem and check_rm_file_version(f):
                ann_rm_file = f
                has_ann = True

        for f in hl_json_files:
            if page_uuid == f.stem:
                hl_json_file = f
                has_smart_hl = True

        # Create a new PDF document to hold the page that will be annotated
        work_doc = fitz.open()

        # Get document page dimensions and calculate what scale should be
        # applied to fit it into the device (given the device's own dimensions)
        pdf_src_dims = (
            pdf_src.load_page(page_idx).rect.width,
            pdf_src.load_page(page_idx).rect.height,
        )
        pdf_src_dims_downscaled, scale = rescale_given_device_aspect_ratio(
            pdf_src_dims,
        )
        # print("pdf_src_dims:", pdf_src_dims)
        # print("scale:", scale)
        # print("pdf_src_dims_downscaled:", pdf_src_dims_downscaled)

        # Create page to annotate using the device's dimensions to allow for
        # "margin" annotations that would be outside the original doc dimensions
        device_dims_downscaled = RM_WIDTH * scale, RM_HEIGHT * scale
        # print("device_dims_downscaled", device_dims_downscaled)

        ann_page = work_doc.new_page(
            width=device_dims_downscaled[0],
            height=device_dims_downscaled[1],
        )

        pdf_src_page_rect = fitz.Rect(
            0, 0, pdf_src_dims_downscaled[0], pdf_src_dims_downscaled[1]
        )

        # This check is necessary because PyMuPDF doesn't let us
        # "show_pdf_page" from an empty (blank) page
        # - https://github.com/pymupdf/PyMuPDF/blob/9d2af43230f6d9944734320813acc79abe95d514/fitz/utils.py#L185-L186
        if len(pdf_src[page_idx].get_contents()) != 0:
            # Resize content of original page and copy it to the page that will
            # be annotated
            ann_page.show_pdf_page(pdf_src_page_rect, pdf_src, pno=page_idx)

            # `show_pdf_page()` works as a way to copy and resize content from
            # one doc/page/rect into another, but unlike `insert_pdf()` it will
            # not carry over in-PDF links, annotations, etc:
            # - https://pymupdf.readthedocs.io/en/latest/page.html#Page.show_pdf_page
            # - https://pymupdf.readthedocs.io/en/latest/document.html#Document.insert_pdf

        is_text_extractable = check_if_text_extractable(
            pdf_src[page_idx],
            malformed=assume_malformed_pdfs,
        )

        is_ann_out_page = False
        ann_data = None

        if "scribbles" in ann_type and has_ann:
            parsed_data, has_ann_hl = parse_rm_file(ann_rm_file)
            # print(parsed_data)

            ann_data = rescale_parsed_data(parsed_data, scale)
            # print(ann_data)

            # Check if there are annotations outside the original page limits
            x_max, y_max = get_ann_max_bound(ann_data)
            is_ann_out_page = (x_max > pdf_src_dims_downscaled[0]) or (
                y_max > pdf_src_dims_downscaled[1]
            )
        # print("is_ann_out_page:", is_ann_out_page)

        if "highlights" not in ann_type and has_ann_hl:
            logging.info(
                "- Found highlighted text on page #{page_idx} but `--ann_type` flag is set to `scribbles` only, so we won't bother with it"
            )

        is_ocred = False

        # This is for highlights that reMarkable's own "smart" detection misses
        # Most likely, they're highlights on scanned / image-based PDF, so in
        # order to extract any text from it, we need to run the PDF through OCR
        #
        # TODO: isn't it faster to run ocr through the whole PDF document at
        # once? (as opposed to doing it per page)
        if (
            doc_type == "pdf"
            and "highlights" in ann_type
            and has_ann_hl
            and not is_text_extractable
            and is_executable_available("ocrmypdf")
            and not avoid_ocr
        ):
            logging.warning("- Will run OCRmyPDF on this document. Hold on!")
            work_doc, ann_page = process_ocr(work_doc)
            is_ocred = True

        if has_ann:
            ann_page = draw_annotations_on_pdf(ann_data, ann_page)

        # TODO: add ability to extract highlighted images / tables (via pixmaps)?

        ann_hl_groups = []
        if (
            "highlights" in ann_type
            and has_ann_hl
            and (is_text_extractable or is_ocred)
        ):
            ann_hl_groups = extract_groups_from_pdf_ann_hl(
                ann_page,
                malformed=assume_malformed_pdfs,
            )
        elif "highlights" in ann_type and has_ann_hl and doc_type == "pdf":
            logging.info(
                f"- Found highlights on page #{page_idx} but couldn't extract them to Markdown. Maybe run it through OCRmyPDF next time?"
            )

        smart_hl_groups = []
        if "highlights" in ann_type and has_smart_hl:
            smart_hl_data = load_json_file(hl_json_file)
            # print("smart_hl_data", smart_hl_data)
            ann_page = add_smart_highlight_annotations(smart_hl_data, ann_page, scale)
            smart_hl_groups = extract_groups_from_smart_hl(smart_hl_data)

        hl_text = ""
        if len(ann_hl_groups + smart_hl_groups) > 0:
            hl_text = prepare_md_from_hl_groups(
                ann_page,
                ann_hl_groups,
                smart_hl_groups,
                presentation=md_hl_format,
            )

        if per_page_targets and (has_ann or has_smart_hl):
            out_path.mkdir(parents=True, exist_ok=True)

            if "pdf" in per_page_targets:
                subdir = prepare_subdir(out_path, "pdf")
                work_doc.save(f"{subdir}/{page_idx:0{pages_magnitude}}.pdf")

            if "png" in per_page_targets:
                # (2, 2) is a short-hand for 2x zoom on (x, y)
                # https://pymupdf.readthedocs.io/en/latest/page.html#Page.get_pixmap
                ann_pixmap = ann_page.get_pixmap(matrix=fitz.Matrix(2, 2))

                subdir = prepare_subdir(out_path, "png")
                ann_pixmap.save(f"{subdir}/{page_idx:0{pages_magnitude}}.png")

            if "svg" in per_page_targets:
                # (2, 2) is a short-hand for 2x zoom on (x, y)
                # https://pymupdf.readthedocs.io/en/latest/page.html#Page.get_svg_image
                ann_svg_str = ann_page.get_svg_image(
                    matrix=fitz.Matrix(2, 2), text_as_path=False
                )

                subdir = prepare_subdir(out_path, "svg")
                with open(f"{subdir}/{page_idx:0{pages_magnitude}}.svg", "w") as f:
                    f.write(ann_svg_str)

            if "md" in per_page_targets:
                subdir = prepare_subdir(out_path, "md")
                with open(f"{subdir}/{page_idx:0{pages_magnitude}}.md", "w") as f:
                    f.write(hl_text)

        if modified_pdf and (has_ann or has_smart_hl):
            mod_pdf.insert_pdf(work_doc, start_at=-1)
            pages_order.append(page_idx)

        if combined_md and (has_ann_hl or has_smart_hl):
            combined_md_strs += [(page_idx + md_page_offset, hl_text + "\n")]

        # If there are annotations outside the original page limits
        # or if the PDF has been OCRed by us, insert the annotated page
        # that we've just (re)created from scratch
        if combined_pdf and (is_ann_out_page or is_ocred):
            pdf_src.insert_pdf(work_doc, start_at=page_idx)
            pdf_src.delete_page(page_idx + 1)

        # Else, draw annotations on the original PDF page (in-place) to do
        # our best to preserve in-PDF links and the original page size
        elif combined_pdf:
            if has_ann:
                draw_annotations_on_pdf(
                    ann_data,
                    pdf_src[page_idx],
                    inplace=True,
                )

            if has_smart_hl:
                add_smart_highlight_annotations(
                    smart_hl_data,
                    pdf_src[page_idx],
                    scale,
                    inplace=True,
                )

        work_doc.close()

    out_doc_path_str = f"{out_path.parent}/{out_path.name}"
    # print("out_doc_path_str:", out_doc_path_str)

    if combined_pdf:
        pdf_src.save(f"{out_doc_path_str} _remarks.pdf")

    if modified_pdf and (doc_type == "notebook" and combined_pdf):
        logging.info(
            "- You asked for the modified PDF, but we won't bother generated it for this notebook. It would be the same as the combined PDF, which you're already getting anyway"
        )
    elif modified_pdf:
        pages_order = sorted(
            range(len(pages_order)),
            key=pages_order.__getitem__,
        )
        # print("pages_order:", pages_order)
        mod_pdf.select(pages_order)
        mod_pdf.save(f"{out_doc_path_str} _remarks-only.pdf")
        mod_pdf.close()

    if combined_md and len(combined_md_strs) > 0:
        combined_md_strs = sorted(combined_md_strs, key=lambda t: t[0])

        if md_header_format == "atx":
            combined_md_str = "".join(
                [f"\n## Page {s[0]}\n\n" + s[1] for s in combined_md_strs]
            )
            combined_md_str = f"# {out_path.name}\n" + combined_md_str

        elif md_header_format == "setex":
            combined_md_str = "".join(
                [f"\nPage {s[0]}\n--------\n" + s[1] for s in combined_md_strs]
            )
            combined_md_str = f"{out_path.name}\n========\n" + combined_md_str

        with open(f"{out_doc_path_str} _highlights.md", "w") as f:
            f.write(combined_md_str)

    pdf_src.close()

def process_ocr(work_doc):
    tmp_fname = "_tmp.pdf"
    work_doc.save(tmp_fname)

    # Note that OCRmyPDF does not recognize handwriting (as of Oct 2022)
    # https://github.com/ocrmypdf/OCRmyPDF/blob/7bd0e43243a05e56a92d6b00fcaa3c826fb3cccd/docs/introduction.rst#L152
    # "- It is not capable of recognizing handwriting."
    tmp_fname = run_ocr(tmp_fname)

    tmp_doc = fitz.open(tmp_fname)

    # Insert the brand new OCRed page as the first one (index=0)
    work_doc.insert_pdf(tmp_doc, start_at=0)
    # Delete the non-OCRed page, now in the second position (index=1)
    work_doc.delete_page(1)
    # Update ann_page reference to the OCRed page
    ann_page = work_doc[0]

    tmp_doc.close()
    pathlib.Path(tmp_fname).unlink()

    return work_doc, ann_page


import argparse
import os

import torch

from lib.config import get_cfg_defaults
from lib.data import make_data_loader
from lib.engine.inference import inference
from lib.modeling import build_model
from lib.utils.comm import synchronize, get_rank
from lib.utils.logger import setup_logger_wandb as setup_logger
from collections import OrderedDict

import os, sys, time, logging, os.path as osp
sys.path.insert(1, f'{os.getcwd()}/utils')
from wandb_utils import Wandb
from config import config, Config
from torch.utils.tensorboard import SummaryWriter

def count_parameters(model):
    train_params = 0
    for name, parameter in model.named_parameters():
        if not parameter.requires_grad: continue
        train_params += parameter.numel()
    print(f"Total Trainable Params: {train_params}")

def main():
    parser = argparse.ArgumentParser(description="VLG")
    parser.add_argument(
        "--config-file",
        default="configs/activitynet.yaml",
        metavar="FILE",
        help="path to config file",
    )
    parser.add_argument(
        "--split",
        type=str,
        default='val',
        choices=['val','test']
    )
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument(
        "--ckpt",
        help="The path to the checkpoint for test, default is the latest checkpoint.",
        default=None,
    )
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )

    args = parser.parse_args()

    num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
    distributed = num_gpus > 1

    if distributed:
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(
            backend="nccl", init_method="env://"
        )
        synchronize()

    cfg = get_cfg_defaults()
    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.SPLIT = args.split
    cfg.CHECKPOINT = args.ckpt

    dataset_name = cfg.DATASETS.TRAIN[0].split('_')[0]
    model_name = osp.splitext(osp.basename(args.ckpt))[0]
    tags = [dataset_name,
            f'{args.split}',
            f'ModelName:{model_name}',
            f'NegProb_{cfg.MODEL.VLG.NEG_PROB}',
            f'NumAggLayers_{cfg.MODEL.VLG.FEATPOOL.NUM_AGGREGATOR_LAYERS}',
            f'NumNeighborsPool_{cfg.MODEL.VLG.FEATPOOL.NUM_NEIGHBOURS}',
            f'NumNeighborsMatch_{cfg.MODEL.VLG.MATCH.NUM_NEIGHBOURS}',
            f'NumIntLayers_{cfg.MODEL.VLG.INTEGRATOR.NUM_AGGREGATOR_LAYERS}',
            f'NumLSTMLayers_{cfg.MODEL.VLG.INTEGRATOR.LSTM.NUM_LAYERS}',
            f'PredictNumLayers_{cfg.MODEL.VLG.PREDICTOR.NUM_STACK_LAYERS}',
            f'LR_{cfg.SOLVER.LR}',
            f'InputStride_{cfg.INPUT.STRIDE}',
            f'EvalStride_{cfg.TEST.STRIDE}',
            f'NMS-th_{cfg.TEST.NMS_THRESH}'
            ]
    cfg.wandb.tags = tags
    cfg.wandb.name = osp.join(osp.basename(cfg.OUTPUT_DIR), model_name)

    # set up logging
    logger = setup_logger(config, cfg)
    logger.info(cfg)

    # init wandb *FIRST*
    if cfg.wandb.use_wandb:
        assert cfg.wandb.entity is not None
        Wandb.launch(cfg, cfg.wandb.use_wandb)
        logging.info(f"Launch wandb, entity: {cfg.wandb.entity}")
    # then init tensorboard
    summary_writer = SummaryWriter(log_dir=f'{cfg.OUTPUT_DIR}/tensorboard')

    if not torch.cuda.is_available() and cfg.MODEL.DEVICE=='cuda':
        cfg.MODEL.DEVICE = 'cpu'

    model = build_model(cfg)
    model.to(cfg.MODEL.DEVICE)
    count_parameters(model)

    output_dir = cfg.OUTPUT_DIR

    #Load checkpoint
    checkpoint = args.ckpt
    if os.path.isfile(checkpoint):
        state_dict = torch.load(checkpoint, map_location=torch.device(cfg.MODEL.DEVICE))['model']
        new_state_dict = OrderedDict()
        for k, v in state_dict.items():
            name = k.replace("module.", "")
            new_state_dict[name] = v
        model.load_state_dict(new_state_dict)

    data_loader, dataset_split = None, None
    if args.split == 'val':
        dataset_split = cfg.DATASETS.VAL
        data_loader = make_data_loader(cfg, is_train=False, is_distributed=distributed, is_for_period=True)

    elif args.split == 'test':
        dataset_split = cfg.DATASETS.TEST
        data_loader = make_data_loader(cfg, is_train=False, is_distributed=distributed)[0]

    else:
        raise ValueError('wrong split')

    with torch.cuda.amp.autocast(enabled=True):
        inference(
            model,
            data_loader,
            dataset_name=dataset_split,
            nms_thresh=cfg.TEST.NMS_THRESH,
            device=cfg.MODEL.DEVICE,
            name=dataset_name,
            logger=logger,
            summary_writer=summary_writer,
            cfg=cfg
        )
        synchronize()

if __name__ == "__main__":
    main()


# -*- coding:utf-8 -*-
import os
import argparse

def resolve_arg():
    parser = argparse.ArgumentParser()
    parser.add_argument("filePath", help="读取文件路径", type=str)
    group = parser.add_mutually_exclusive_group(**{"required": True})
    group.add_argument("-hex", help="十六进制输出", action="store_true")
    group.add_argument("-utf8", "-u", help="utf8输出", action="store_true")
    group.add_argument("-rawutf8", "-r", help="原生utf8输出", action="store_true")
    return parser.parse_args()

def hexrepr(byte):
    return "{0:02x}".format(byte)

if __name__ == '__main__':
    argv = resolve_arg()
    filePath = argv.filePath
    if not os.path.exists(filePath):
        print("未知文件")
        exit(1)
    with open(filePath, mode="rb") as file:
        content = file.read()
        try:
            if argv.hex:
                print(" ".join(map(hexrepr, content)))
            if argv.utf8:
                print(content.decode("utf8"))
            if argv.rawutf8:
                print(repr(content.decode("utf8")))
        except Exception as e:
            print(e)