from bootstrap_datepicker_plus import DatePickerInput
from django import forms
from django.contrib.auth.models import User
from django.core import validators
from django.shortcuts import get_object_or_404
from django.utils.translation import gettext_lazy as _
from main_app.models import Region
from places.models import Place
from photo_reports.models import PhotoReport, Photo
from video_reports.models import VideoReport

class PhotoReportForm(forms.ModelForm):
    """ PhotoReport Form """
    class Meta:
        model = PhotoReport
        fields = [
            'author',
            'authors',
            'place',
            'date',
            'title',
            'pay_to_photographer',
            'pay_to_portal',
            'is_plus_18',
            'is_posted',
        ]

    def __init__(self, request, *args, **kwargs):
        super(PhotoReportForm, self).__init__(*args, **kwargs)
        region = get_object_or_404(Region, slug=request.session['user_region'])
        authors = User.objects.filter(
            user_access__region=region, user_access__is_photographer=True)
        places = Place.objects.filter(region=region, is_closed=False)

        AUTHOR_CHOICES = []
        for author in authors:
            if author.get_full_name():
                AUTHOR_CHOICES.append((str(author.id), author.get_full_name()))
            else:
                AUTHOR_CHOICES.append((str(author.id), author.get_username()))

        self.fields['author'].choices = AUTHOR_CHOICES
        self.fields['author'].label = _('Primary Photographer')

        self.fields['authors'].choices = AUTHOR_CHOICES
        self.fields['authors'].label = _('More Photographers')
        self.fields['authors'].help_text = _('Hold down "Control", or "Command" on a Mac, to select more than one.')

        PLACE_CHOICES = [(str(place.id), place.name) for place in places]
        self.fields['place'].choices = PLACE_CHOICES

        if request.LANGUAGE_CODE == 'en':
            self.fields['date'].widget = DatePickerInput(format='%Y-%m-%d')
        elif request.LANGUAGE_CODE == 'ru' or request.LANGUAGE_CODE == 'uk':
            self.fields['date'].widget = DatePickerInput(format='%d.%m.%Y')

class PhotosForm(forms.ModelForm):
    """ Photos Form """
    image = forms.ImageField(
        label=_('Image'),
        validators=[
            validators.FileExtensionValidator(
                allowed_extensions=('jpg', 'png'))],
        error_messages={
            'invalid_extension': _('This file format is not supported.')},
        widget=forms.widgets.ClearableFileInput(attrs={'multiple': True}),
        required=False)

    class Meta:
        model = Photo
        fields = ['image']

class VideoReportForm(forms.ModelForm):
    """ VideoReport Form """
    image = forms.ImageField(
        label=_('Image'),
        help_text=_('The minimum image size should be 477x373!'),
        validators=[
            validators.FileExtensionValidator(
                allowed_extensions=('jpg', 'png'))],
        error_messages={
            'invalid_extension': _('This file format is not supported.')})

    class Meta:
        model = VideoReport
        fields = [
            'author',
            'place',
            'date',
            'title',
            'pay_to_photographer',
            'pay_to_portal',
            'image',
            'url_address',
        ]

    def __init__(self, request, *args, **kwargs):
        super(VideoReportForm, self).__init__(*args, **kwargs)
        region = get_object_or_404(Region, slug=request.session['user_region'])
        authors = User.objects.filter(
            user_access__region=region, user_access__is_photographer=True)
        places = Place.objects.filter(region=region, is_closed=False)

        AUTHOR_CHOICES = []
        for author in authors:
            if author.get_full_name():
                AUTHOR_CHOICES.append((str(author.id), author.get_full_name()))
            else:
                AUTHOR_CHOICES.append((str(author.id), author.get_username()))
        self.fields['author'].choices = AUTHOR_CHOICES

        PLACE_CHOICES = [(str(place.id), place.name) for place in places]
        self.fields['place'].choices = PLACE_CHOICES

        if request.LANGUAGE_CODE == 'en':
            self.fields['date'].widget = DatePickerInput(format='%Y-%m-%d')
        elif request.LANGUAGE_CODE == 'ru' or request.LANGUAGE_CODE == 'uk':
            self.fields['date'].widget = DatePickerInput(format='%d.%m.%Y')


'''
    FRANCESCO GERRATANA WWW.NEXTECHNICS.COM 2020
    Scraping with python Bing COVID-19.
'''
import urllib
import requests
import json
from operator import itemgetter
from tabulate import _table_formats, tabulate
import sys
import os
import subprocess

#Afghanistan
#Albania
#Algeria
#Andorra
#Angola
#Anguilla
#Antigua e Barbuda
#Arabia Saudita
#Argentina
#Armenia
#Aruba
#Australia
#Austria
#Azerbaigian
#Bahamas
#Bahrein
#Bangladesh
#Barbados
#Belgio
#Belize
#Benin
#Bermuda
#Bhutan
#Bielorussia
#Bolivia
#Bonaire
#Bosnia ed Erzegovina
#Botswana
#Brasile
#Brunei
#Bulgaria
#Burkina Faso
#Burundi
#Cabo Verde
#Cambogia
#Camerun
#Canada
#Cayman
#Cechia
#Ciad
#Cile
#Cina (continentale)
#Cipro
#Cisgiordania
#Città del Vaticano
#Colombia
#Congo
#Congo (RDC)
#Corea del Sud
#Costa Rica
#Croazia
#Cuba
#Curaçao
#Côte d’Ivoire (Costa d’Avorio)
#Danimarca
#Dominica
#Ecuador
#Egitto
#El Salvador
#Emirati Arabi Uniti
#Eritrea
#Estonia
#Etiopia
#Figi
#Filippine
#Finlandia
#Francia
#Gabon
#Gambia
#Georgia
#Germania
#Ghana
#Giamaica
#Giappone
#Gibilterra
#Gibuti
#Giordania
#Grecia
#Grenada
#Groenlandia
#Guadalupa
#Guam
#Guatemala
#Guernsey
#Guinea
#Guinea Equatoriale
#Guinea-Bissau
#Guyana
#Guyana francese
#Haiti
#Honduras
#India
#Indonesia
#Iran
#Iraq
#Irlanda
#Islanda
#Isola di Man
#Isole Falkland
#Isole Fær Øer
#Isole Marianne settentrionali
#Isole Turks e Caicos
#Isole Vergini Americane
#Isole Vergini Britanniche
#Israele
#Italia
#Jersey
#Kazakistan
#Kenya
#Kirghizistan
#Kosovo
#Kuwait
#Laos
#Lettonia
#Libano
#Liberia
#Libia
#Liechtenstein
#Lituania
#Lussemburgo
#Macedonia del Nord
#Madagascar
#Malawi
#Malaysia
#Maldive
#Mali
#Malta
#Marocco
#Martinica
#Mauritania
#Mauritius
#Mayotte
#Messico
#Moldova
#Monaco
#Mongolia
#Montenegro
#Montserrat
#Mozambico
#Myanmar
#Namibia
#Nepal
#Nicaragua
#Niger
#Nigeria
#Norvegia
#Nuova Caledonia
#Nuova Zelanda
#Oman
#Paesi Bassi
#Pakistan
#Panamá
#Papua Nuova Guinea
#Paraguay
#Perù
#Polinesia francese
#Polonia
#Portogallo
#Portorico
#Qatar
#RAS di Hong Kong
#RAS di Macao
#Regno Unito
#Repubblica Centrafricana
#Repubblica Dominicana
#Riunione
#Romania
#Ruanda
#Russia
#Saba
#Saint Lucia
#Saint Martin
#Saint-Barthélemy
#Saint-Pierre e Miquelon
#San Marino
#Senegal
#Serbia
#Seychelles
#Sierra Leone
#Singapore
#Siria
#Slovacchia
#Slovenia
#Somalia
#Spagna
#Sri Lanka
#Stati Uniti
#Sud Sudan
#Sudafrica
#Sudan
#Suriname
#Svezia
#Svizzera
#São Tomé e Príncipe
#Taiwan
#Tanzania
#Thailandia
#Timor Est
#Togo
#Trinidad e Tobago
#Tunisia
#Turchia
#Ucraina
#Uganda
#Ungheria
#Uruguay
#Uzbekistan
#Vanuatu
#Venezuela
#Vietnam
#Yemen
#Zambia
#Zimbabwe
#eSwatini

#response = requests.get("https://www.bing.com/covid/data")
#url = json.loads(response.text)

def covidscrapy(data,location):

    my_list = []
    totc, totd, totr= 0,0,0
    tc, td, tr= 0,0,0
    totp = 0
    st, sn = "",""
    somma,sommad,sommag = 0,0,0
    totpd = 0

    for each in data['areas']:
      if each['displayName'] == location:
        st = each['displayName']
        totc = each['totalConfirmed']
        totd = each['totalDeaths']
        totr = each['totalRecovered']
        totp = (each['totalConfirmed']-(each['totalDeaths']+each['totalRecovered']))
        my_list.append((st,"",totc,totd,totr,totp))
        for x in each['areas']:
           sn = x['displayName']
           tc = x['totalConfirmed']
           td = x['totalDeaths']
           tr = x['totalRecovered']

          #gestisco il valore None
           if tc is not None and td is not None and tr is not None:
                totpd = (tc-(td+tr))#si si si
           elif tc is None and td is not None and tr is not None:
                totpd = (td+tr)#no si si
           elif tc is None and td is None and tr is not None:
                totpd = (td+tr)#no no si
           elif tc is None and td is None and tr is None:
                totpd = (0)#no no no
           elif tc is None and td is not None and tr is None:
                totpd = (td)#no si no
           elif tc is not None and td is not None and tr is None:
                totpd = (tc-(td+0))#si si no
           elif tc is not None and td is None and tr is not None:
                totpd = (tc-(0+tr))#si no si

           #calcolo manuale
           if x['totalConfirmed'] is not None:
               somma = somma + x['totalConfirmed']
           if x['totalDeaths'] is not None:
               sommad = sommad + x['totalDeaths']
           if x['totalRecovered'] is not None:
               sommag = sommag + x['totalRecovered']

           my_list.append(("",sn,tc,td,tr,totpd))

        #calcolo manuale
        my_list.append(("Somma","",somma,sommad,sommag,somma-(sommad+sommag)))

    return sorted(my_list, key=itemgetter(2),reverse=True)

url = json.load(open('bing.json', encoding='utf-8'))

db = covidscrapy(url,input("Enter state: ").title())

header = ['STATO','REGIONE',' TOT CONFERMATI',' TOT DECESSI',' TOT GUARITI','TOT POSITIVI']
print(tabulate(db,header,tablefmt="grid"))

with open("file.txt", "w",encoding='utf-8') as output:
    output.write(str(tabulate(db,header,tablefmt="grid")))

subprocess.call(["cmd", "/k", "start", "", "file.txt"], stderr=subprocess.STDOUT)


import argparse
import datetime
import errno
import ntpath
import os
import pickle
import random
import shutil
import string
import sys
import time
import traceback
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from datetime import datetime
from itertools import islice

import pandas as pd

def current_time():
    return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]

def test(write, file_name, output_dir, data, chunk_number):
    print('batch {}: running  with  pid {}'.format(chunk_number, os.getpid()))
    even = []
    odd = []
    for line in data:
        if int(line) % 2 == 0:
            even.append("{}\n".format(line))
        else:
            odd.append("{}\n".format(line))
    all_groups = [even, odd]
    write(all_groups, file_name, output_dir, chunk_number)
    return len(data)

def split_dummy_test(file_name, output_dir, data, chunk_number, params):
    print(file_name, output_dir, chunk_number, params)

def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

def ngram5_validation(file_name, output_dir, data, chunk_number, params):
    for line in data:
        sys.stdout.write("\r sa->" + random.choice(string.ascii_letters))
        sys.stdout.flush()
        line = line.lower()
        words = line.split()
        if len(words) != 8:
            print("ngram is not 8 words => {}".format(line))

        if not (len(words) > 5) & (is_int(words[5])):
            print("year {} is not valid for ==> {}".format(words[5], line))

def split_data_by_word_test_case():
    moods_dict = prepare_moods_dict("E:/download/proj/n5grm/small_ds/emotion-lexicon-master/eng/")
    data = ["1947-1958 ( islamabad : threat	2008	2	2",
            "1992_num )_. the_det islamic_adj threat_noun	1995	7	7",
            "1035_noun ,_. islamabad_adj ;_. _end_	1970	2	2"]
    split_data_by_word("fname.txt", moods_dict, "debug_output/", data, 0, 0)

def split_data_by_word(file_name, moods_dict, can_check_pos, output_dir, lines, chunk_number, isparallel, params):
    # print('process started', file_name, chunk_number, len(data))
    my_word_list = moods_dict  # {'muslim': ['one', 'two'], 'islam': ['three', 'four']}
    words_result = []
    mixed_polarity_key = 'mixed_polarity'
    nutral_key = 'neutral'
    mood_year_wise = {}
    year_index_in_ngram = 5
    n5grm_length = 8
    pos_check = "yes" in can_check_pos
    no_of_lines = len(lines)
    islamic_words = ['muslim', 'islam', 'muslims', "muslim's"]
    mood_year_wise_word_wise = {}
    for line in lines:
        sys.stdout.write("\r{}=>{}==>{}".format(random.choice(string.ascii_letters), chunk_number, (no_of_lines)))
        sys.stdout.flush()
        line = line.lower()
        no_of_lines = no_of_lines - 1
        # validate the ngram data
        ngrams = line.split()
        if len(ngrams) != n5grm_length:
            break
        year = 0
        if is_int(ngrams[year_index_in_ngram]):
            year = ngrams[year_index_in_ngram]
        if year == 0:
            break
        if not is_int(ngrams[6]):
            break

        no_of_occ = int(ngrams[6])

        if year not in mood_year_wise:
            mood_year_wise[year] = {}
        # the count
        # islamic words count
        #the_islamic_counts(islamic_words, line, mood_year_wise, ngrams, year, pos_check)

        # word moods counts year wise
        # words count year wise
        # add counts to temp group, later we will find how many groups and mixed or nuteral
        mood_group_wise_count = {}
        for k_group, v_moods in my_word_list.items():
            for mood in v_moods:
                local_word = 0
                if mood in line:
                    for ngram in ngrams[0:5]:
                        pos_found = False
                        if pos_check:
                            pos_found = pos_in_word(ngram)
                            if pos_found & pos_check:
                                continue

                        if (mood == ngram) & pos_check & (not pos_found):
                            local_word = local_word + 1
                            # print("pos found ==>"+k_group+"==> " + mood +"==> "+ str(local_word))
                        elif (mood == ngram) & (not pos_check):
                            local_word = local_word + 1
                        if local_word > 0:
                            if year not in mood_year_wise_word_wise:
                                mood_year_wise_word_wise[year] = {}
                            if k_group not in mood_year_wise_word_wise[year]:
                                mood_year_wise_word_wise[year][k_group] = {}
                            if mood not in mood_year_wise_word_wise[year][k_group]:
                                mood_year_wise_word_wise[year][k_group][mood] = local_word
                            else:
                                mood_year_wise_word_wise[year][k_group][mood] = mood_year_wise_word_wise[year][k_group][
                                                                                mood] + local_word

                if local_word == 0:
                    continue
                local_word = local_word * no_of_occ

                if k_group in mood_group_wise_count:
                    mood_group_wise_count[k_group] = mood_group_wise_count[k_group] + local_word
                else:
                    mood_group_wise_count[k_group] = local_word

        # count mood wise count to see if the diffent moods are availae or single mood
        c = sum(mood_group_wise_count.values())
        if c > 0:
            pass
            # print("c count ==> " +str(c))
        # seperated ds for moods related lines
        if c > 0:
            words_result.append(line)

        # if ngram not contains any mood related word
        if len(mood_group_wise_count) == 0:
            k_group = nutral_key
            c = 1
        elif len(mood_group_wise_count) > 1:
            k_group = mixed_polarity_key
        else:
            k_group = list(mood_group_wise_count)[0]

        if k_group not in mood_year_wise[year]:
            mood_year_wise[year][k_group] = 0

        mood_year_wise[year][k_group] = mood_year_wise[year][k_group] + c

    suffix = '_with_out_pos'
    if pos_check:
        suffix = '_with_pos'

    if len(words_result) > 0:
        write_to_file(words_result, file_name, output_dir + "moods_words_ds" + suffix, chunk_number)

    if len(mood_year_wise) > 0:
        fpath = write_to_file_pickle(mood_year_wise, file_name, output_dir + "moods_year_wise_count" + suffix,
                                     chunk_number)
        write_to_file_pickle(mood_year_wise_word_wise, file_name + "count_mood_word_wise",
                             output_dir + "moods_year_wise_count" + suffix, chunk_number)
        print(fpath)

        # if not parallel, then it is single file, we write it here
        if not isparallel:
            moods_output_pickle_to_excel(fpath, os.path.join(os.path.dirname(fpath), "moods_year_wise_count.xlsx"))

    return file_name, chunk_number, len(lines), params

def moods_output_pickle_to_excel(input_file, output_file):
    moods = pd.read_pickle(input_file)
    df = pd.DataFrame.from_dict(moods, orient='index')
    df = df.sort_index(axis=1)
    df.fillna(0, inplace=True)
    df.to_excel(output_file)
    df.to_csv("{}.csv".format(output_file))

def the_islamic_counts(islamic_words, line, mood_year_wise, ngrams, year, pos_check):
    if 'the' or 'muslim' or 'islam' in line:

        mood_year_wise[year]['the'] = 0
        mood_year_wise[year]['islam'] = 0

        the_count = 0
        islamic_count = 0
        for ngram in ngrams[0:5]:
            if pos_check:
                if pos_in_word(ngram):
                    continue
            if 'the' == ngram:
                the_count = the_count + 1
            else:
                for isl in islamic_words:
                    if isl == ngram:
                        islamic_count = islamic_count + 1

        if the_count > 0:
            mood_year_wise[year]['the'] = mood_year_wise[year]['the'] + the_count
        if islamic_count > 0:
            mood_year_wise[year]['islam'] = mood_year_wise[year]['islam'] + the_count

def pos_in_word(w):
    pos = ['adj', 'noun', 'adp', 'adv', 'conj', 'det', 'num', 'pron', 'prt', 'verb']
    for p in pos:
        if p in w:
            pos_word = w.split('_')
            for pow in pos_word:
                if p == pow:
                    return True
    return False

def write_to_file_pickle(data, file_name, output_dir, chunk_number):
    if len(data) == 0:
        return ""

    dir_path = os.path.join(output_dir, os.path.basename(file_name))
    try:
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    path = os.path.join(dir_path, "{}.part.{}.pickle".format(current_time(), str(chunk_number)))
    if (os.path.exists(path)):
        print("file path exit {}".format(path))
    path = path.replace("\\\\", "\\")

    with open(path, 'wb') as f:
        # Pickle the 'data' dictionary using the highest protocol available.
        pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
    return path

def write_to_file_text(data, file_name, output_dir, chunk_number):
    if len(data) == 0:
        return ""

    dir_path = os.path.join(output_dir, os.path.basename(file_name))
    try:
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    path = os.path.join(dir_path, "{}.part.{}.txt".format(current_time(), str(chunk_number)))
    if (os.path.exists(path)):
        print("file path exit {}".format(path))
    path = path.replace("\\\\", "\\")

    with open(path, 'w') as f:
        f.write(data)
    return path

def write_to_file_callback(future):
    print('files written to disk')

def time_milli_sec():
    x = lambda: int(round(time.time() * 1000))
    return x()

def write_to_file(lines, file_name, output_dir, chunk_number):
    if len(lines) == 0:
        return True
    dir_path = os.path.join(output_dir, os.path.basename(file_name))
    try:
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise
    path = os.path.join(dir_path, "{}.part.{}".format(current_time(), str(chunk_number)))
    if os.path.exists(path):
        print("file path exit {}".format(path))
    path = path.replace("\\\\", "\\")

    with open(path, 'w') as fs:
        fs.writelines(lines)
    return True

def callback_function(future):
    # print('process finished', future.result())
    x = future.result()

def chunks(list_values, chunk_size):
    chunk_size = max(1, chunk_size)
    return (list_values[i:i + chunk_size] for i in range(0, len(list_values), chunk_size))

def start_process():
    global parser, args
    parser = argparse.ArgumentParser(description='Parallel text processor')
    parser.add_argument('--num_workers', '-n', default=8, type=int)
    parser.add_argument('--chunk_size', '-c', default=8, type=int)
    parser.add_argument('--no_of_file_in_parallel', '-f', default=8, type=int)
    parser.add_argument('--output_dir', type=str, required=True)
    parser.add_argument('--input_dir', type=str, required=True)
    parser.add_argument('--moods_dir', type=str, required=True)
    parser.add_argument('--can_check_pos', type=str, required=True)

    args = parser.parse_args()
    main(args.num_workers, args.chunk_size, args.no_of_file_in_parallel, args.input_dir, args.output_dir,
         args.moods_dir, args.can_check_pos)

def get_file_size(file_path):
    size = os.path.getsize(file_path)
    return size

def convert_bytes(size, unit=None):
    if unit == "KB":
        return ('File size: ' + str(round(size / 1024, 3)) + ' Kilobytes')
    elif unit == "MB":
        return ('File size: ' + str(round(size / (1024 * 1024), 3)) + ' Megabytes')
    elif unit == "GB":
        return ('File size: ' + str(round(size / (1024 * 1024 * 1024), 3)) + ' Gigabytes')
    else:
        return ('File size: ' + str(size) + ' bytes')

def write_dict_as_csv(dic, file_name):
    pass

def write_dict_as_pickle(dic, file_name):
    with open(file_name, 'wb') as f:
        pickle.dump(dic, f, pickle.HIGHEST_PROTOCOL)

def write_text_to_file(content, file_name):
    with open(file_name, 'w') as f:
        f.writelines(content)

def main(num_workers, chunk_size, no_of_file_in_parallel, input_dir, output_dir, moods_input_dir, can_check_pos):
    print("{} workers".format(num_workers))
    if not os.path.exists(output_dir):
        print("output will be here {}", output_dir)
        os.makedirs(output_dir)
    # for each input file
    files = os.listdir(input_dir)
    file_worker = no_of_file_in_parallel

    print("preparing moods dictionary from {}".format(moods_input_dir))
    moods_dict = prepare_moods_dict(moods_input_dir, output_dir)

    write_text_to_file(str(moods_dict), os.path.join(output_dir, "moods_dict.txt"))
    write_dict_as_pickle(moods_dict, os.path.join(output_dir, "moods_dict.pkl"))

    if (num_workers <= 1) or (chunk_size == 0):
        print("programe is running in single thread mode")
        for file_name in files:
            print("processing file ".format(file_name))
            process_file_no_parallel(0, file_name, input_dir, 1, output_dir, moods_dict, can_check_pos)
    else:
        file_futures = []
        with ProcessPoolExecutor(max_workers=file_worker) as fe:
            for file_name in files:
                file_future = fe.submit(process_file, chunk_size, file_name, input_dir, num_workers, output_dir,
                                        moods_dict, can_check_pos)
                file_futures.append(file_future)
            file_results = [f.result() for f in file_futures]
            # print(file_results)
            # process_file(chunk_size, file_name, input_dir, num_workers, output_dir)

def prepare_moods_dict(moods_input_dir, output_dir):
    moods_dict = []
    for f in os.listdir(moods_input_dir):
        paath = os.path.join(moods_input_dir, f)
        print("reading dictionary {}".format(paath))
        if os.path.isfile(paath):
            (fn, ext) = os.path.splitext(f)
            print("preparing mood dict for mood type {}".format(fn))
            with open(paath) as ff:
                emotions = {}
                mood_l = ff.readline().split()
                mood_l = list(set(mood_l))
                emotions[fn] = mood_l
                moods_dict.append(emotions)
                print("{} has {} words".format(fn, len(mood_l)))

    dict_duplicate, lst_duplicates = find_duplicate_moods_actross_all_emotions(moods_dict)
    print("{} been duplicate removed from dict", len(lst_duplicates))
    write_dict_as_pickle(dict_duplicate, os.path.join(output_dir, "duplicate_moods_across_all_moods.pickle"))
    write_dict_as_pickle(list(lst_duplicates), os.path.join(output_dir, "duplicate_moods_words_list.pickle"))

    # removing duplicate moods from all emotions
    uniq_dict = {}
    for i in range(0, len(moods_dict)):
        dic1 = moods_dict[i]
        for k, v in dic1.items():
            for d in lst_duplicates:
                try:
                    v.remove(d)
                except ValueError:
                    pass

        uniq_dict[k] = set(v)

    return uniq_dict

def find_duplicate_moods_actross_all_emotions(moods_dict):
    duplicate_cross_moods = {}
    all_duplicate_words = set()
    for x in range(0, len(moods_dict)):
        dic1 = moods_dict[x]
        for y in range(0, len(moods_dict)):
            dp = []
            if x == y:
                continue
            dic2 = moods_dict[y]

            for k, v in dic1.items():
                for a, b in dic2.items():
                    match = set(v) & set(b)

                    if (len(match) > 0):
                        dp.append(match)
                        all_duplicate_words.update(match)
            duplicate_cross_moods["{}_{}".format(k, a)] = dp

    return duplicate_cross_moods, all_duplicate_words

def process_file(chunk_size, file_name, input_dir, num_workers, output_dir, moods_dict, can_check_pos):
    futures = []
    try:
        file_name = os.path.join(input_dir, file_name)
        size = get_file_size(file_name)
        print("params ==>", chunk_size, file_name, input_dir, num_workers, output_dir)
        print("processing file {}, size {} <=> size{}, started at {}".format(file_name, convert_bytes(size, "MB"),
                                                                             convert_bytes(size, "GB"),
                                                                             datetime.utcnow()))
        chunk_number = 0
        print("programme running in parrllel mode")

        with open(file_name) as file:
            lines = get_lines(file,40000)  # list(islice(file, chunk_size))
            print("total lines are {}".format(len(lines)))
            # batch_size = int(chunk_size / num_workers)
            chunked_data = chunks(lines, chunk_size)
            with ProcessPoolExecutor(max_workers=num_workers) as e:
                for data in chunked_data:
                    print("{} chunk started in parallel".format(chunk_number))
                    # split_data_by_word(file_name, moods_dict, can_check_pos, output_dir, lines, 0,False,("started at {}".format(current_time()), 0))
                    future = e.submit(split_data_by_word, file_name, moods_dict, can_check_pos, output_dir, data,
                                      chunk_number, True, ("started at {}".format(current_time()), chunk_number))
                    future.add_done_callback(callback_function)
                    futures.append(future)
                    chunk_number = chunk_number + 1

        results = [f.result() for f in futures]
        suffix = '_with_out_pos'
        if "yes" in can_check_pos:
            suffix = '_with_pos'
        print("ouput folder {}".format(output_dir))
        temp_dir = os.path.join(output_dir , "moods_year_wise_count" + suffix,"words_ds_is_mu")
        print("temp folder {}".format(temp_dir))
        output_file = os.path.join(output_dir, "moods_year_wise_count.pickle")
        print("output file  {}".format(output_file))
        #mood year wise
        merge_and_update_counts_dict_from_pickle(temp_dir, output_file)
        moods_output_pickle_to_excel(output_file, os.path.join(output_dir, "moods_year_wise_count.xlsx"))

        #merging dataset
        temp_dir = os.path.join(output_dir, "moods_words_ds" + suffix, "words_ds_is_mu")
        merge_files_in_folder_shutil(temp_dir,output_dir,'dataset'+suffix)
        output_file = os.path.join(output_dir, "moods_word_wise_year_wise_count.pickle")

        # word wise count
        temp_dir = os.path.join(output_dir, "moods_year_wise_count" + suffix, "words_ds_is_mucount_mood_word_wise")
        merge_and_update_counts_nested_dict_from_pickle(temp_dir, output_file)
        moods_output_pickle_to_excel(output_file, os.path.join(output_dir, "moods_word_wise_year_wise_count.xlsx"))
        # total_lns = np.sum(results)  # [count(col) for col in results]
        # print("total lines {}".format(total_lns))
        print("{} finished at {}".format(file_name, datetime.utcnow()))
    except Exception as e:
        traceback.print_exc()

def merge_files_in_folder_shutil(input_floder, output_folder,output_filename):
    outfilename = os.path.join(output_folder, 'all_' + str((int(time.time()))))

    with open(outfilename, 'wb') as outfile:
        for filename in os.listdir(input_floder):
            if filename == outfilename:
                # don't want to copy the output into the output
                continue
            with open(os.path.join(input_floder,filename), 'rb') as readfile:
                shutil.copyfileobj(readfile, outfile)
    print("finishd..")

def path_leaf(path):
    head, tail = ntpath.split(path)
    return tail or ntpath.basename(head)

def merge_and_update_counts_dict_from_pickle(input_folder, ouput_file_name):
    print("in merge dict , input folder "+input_folder)
    pkl_file_list = os.listdir(input_folder)
    #pkl_file_list = os.listdir(os.path.join(input_folder,pkl_file_list[0]))
    print("list dir finished")
    All_dict = Counter({})
    for pkl_file in pkl_file_list:

        pkl_file = os.path.join(input_folder, pkl_file)
        print("pickle file "+pkl_file)
        if os.path.exists(pkl_file):
            with  open(pkl_file, "rb") as pickle_in:
                dict_i = pickle.load(pickle_in)
                for k, v in dict_i.items():
                    All_dict = All_dict + Counter(v)

    with  open(ouput_file_name, "wb") as pickle_out:
        pickle.dump(dict(All_dict), pickle_out)
        pickle_out.close()

def merge_and_update_counts_nested_dict_from_pickle(input_folder, ouput_file_name):
    print("in merge dict , input folder "+input_folder)
    pkl_file_list = os.listdir(input_folder)
    #pkl_file_list = os.listdir(os.path.join(input_folder,pkl_file_list[0]))
    print("list dir finished")
    All_dict = Counter({})
    for pkl_file in pkl_file_list:

        pkl_file = os.path.join(input_folder, pkl_file)
        print("pickle file "+pkl_file)
        if os.path.exists(pkl_file):
            with  open(pkl_file, "rb") as pickle_in:
                dict_i = pickle.load(pickle_in)
                for k, v in dict_i.items():
                    for kk,vv in v.items():
                        All_dict = All_dict + Counter(vv)

    with  open(ouput_file_name, "wb") as pickle_out:
        pickle.dump(dict(All_dict), pickle_out)
        pickle_out.close()

def process_file_no_parallel(chunk_size, file_name, input_dir, num_workers, output_dir, moods_dict, can_check_pos):
    output = []
    try:
        file_name = os.path.join(input_dir, file_name)
        size = get_file_size(file_name)
        print("processing file {}, size {} <=> size{}, started at {}".format(file_name, convert_bytes(size, "MB"),
                                                                             convert_bytes(size, "GB"),
                                                                             datetime.utcnow()))
        chunk_number = 0
        lines = []
        line_number = 0
        with open(file_name) as file:
            lines = get_lines(file)

            res = split_data_by_word(file_name, moods_dict, can_check_pos, output_dir, lines, 0, False,
                                     ("started at {}".format(current_time()), 0))

        print("{} finished at {}".format(file_name, datetime.utcnow()))
    except Exception as e:
        traceback.print_exc()

def get_lines(file, no_of_lines_to_read=0):
    if no_of_lines_to_read == 0:
        return file.readlines()

    lines = []
    line_number = 0
    for line in file:
        lines.append(line)
        line_number = line_number + 1
        if (line_number == no_of_lines_to_read):
            break
    print("no of lines {}".format(line_number))
    return lines

# num_workers, chunk_size, no_of_file_in_parallel, input_dir, output_dir, moods_input_dir, can_check_pos
# main(10, 500000, 10, 'E:/download/proj/n5grm/small_ds/data/moods_ds/', "E:/download/proj/n5grm/small_ds/data/output",     "E:/download/proj/n5grm/small_ds/emotion-lexicon-master/eng_no_brotherhood/", "yes")

if __name__ == '__main__':
    # test_batch_execution('one_file', "E:/download/output_files")#
    # split_data_by_word_test_case()
    start_process()
    # prepare_moods_dict("E:\\download\\proj\\n5grm\\small_ds\\emotion-lexicon-master\\eng_no_brotherhood\\")
    print("in main")


import pytest
import requests

resp= requests.get("https://reqres.in/api/users?page=2")
code = resp.status_code
assert code == 200, "code doesn't match"

json_resp = resp.json()
print(json_resp)

@pytest.mark.one
def test_json_resp():
    assert json_resp['total']==12
@pytest.mark.two
def test_json_resp_1():
    assert json_resp['total_pages'] == 2, "mismatch ib total pges"


# -*- coding: utf-8 -*-
"""
Created on Sat Nov 19 18:36:46 2016

@author: User
"""
class A:
    def __init__(self,name,age):
        self._name = name
        self._age = age

class B:
    def __init__(self,AList):
        for i in range(len(AList)):
            AList
            a = AList[i]
            print(a._name,a._age)
'''
a1 = A("Rose",24)
a2 = A("Mike",15)
Alist = list()
Alist.append(a1)
Alist.append(a2)
B(Alist)
'''
print([0,])


import sys

def parse_input(puzzle_input):
    pairs = []
    for line in puzzle_input.splitlines():
        raw_fst, raw_snd = line.split(",")
        fst = tuple(int(x) for x in raw_fst.split("-"))
        snd = tuple(int(x) for x in raw_snd.split("-"))

        pairs.append((fst, snd))

    return (pairs,)

def part_one(pairs):
    total = 0
    for (a, b), (c, d) in pairs:
        total += (a <= c <= d <= b) or (c <= a <= b <= d)

    return total

def part_two(pairs):
    total = 0
    for (a, b), (c, d) in pairs:
        total += (a <= c <= b) or (c <= a <= d)

    return total

class Test:
    example = """\
2-4,6-8
2-3,4-5
5-7,7-9
2-8,3-7
6-6,4-6
2-6,4-8
"""

    def test_one(self):
        assert part_one(*parse_input(self.example)) == 2

    def test_two(self):
        assert part_two(*parse_input(self.example)) == 4

def main():
    puzzle = parse_input(sys.stdin.read())

    print("part 1:", part_one(*puzzle))
    print("part 2:", part_two(*puzzle))

if __name__ == "__main__":
    sys.exit(main())


"""
将SQL翻译成pandas的操作
SQL格式(类MySQL)：

update table_1 a
   set a.name=b.name,
       a.age=a.score+3,
       a.level=my_func(b.level, a.id)
  left join table_2 b    -- 使用join 或left join关联
    on a.id=b.id         -- 翻译成 pandas_merge 的 left_on 和 right_on
 where a.age>3              -- 支持比较操作
   and b.score<=3           -- 支持比较操作
   and a.score>b.score      -- 支持两边对比操作
   and a.id is in (1,2,3)   -- 支持 isin 操作
   and b.id not in (1,2,3)  -- 支持 not in 操作
   and a.score <> 100       -- 支持不等于操作
   and a.id is not null     -- 支持 not null
   and b.level is null      -- 支持 is null
支持操作：
1、带where条件的单表update
2、带where条件的两个表update
3、自定义python函数

不支持的操作：
1、like，比如 string like '%abc%
4、SQL中对字段进行字符串操作，比如 replace(a.name,'小明','小红')

基本假设：
1、SQL中没有对 '\n' 的操作，比如替换，因为在解析SQL时会将换行删除
2、只支持2个表的join操作，不支持3个及以上表关联
3、不对字符字段进行操作，即操作涉及到的列没有字符型，但是支持 =,!=,isin, not in
4、不支持case when 等复杂语句

"""
import re
import sqlparse
import pandas as pd

def judge_format(sql, table_1_cols, table_2_cols):
    """sql校验"""
    # 去掉换行符,多个空格只保留1个
    sql = re.compile(r'\s{2,100}', flags=re.I | re.S).subn(' ', sql.replace('\n', ' '))[0].strip()
    # 判断是否有表别名
    # update table_1 a set xxx
    tb1 = re.compile(r"update (.+?) (.+?) set", flags=re.I | re.S).findall(sql)
    if len(tb1) == 0:
        raise Exception('update表没有表别名，请检查')
    # left join table_2 b on
    if ' left join ' in sql and ' on ' in sql:
        tb2 = re.compile(r" left join (.+?) (.+?) on", flags=re.I | re.S).findall(sql)
        if len(tb2) == 0:
            raise Exception('join表没有表别名，请检查')
    elif ' join ' in sql and ' on ' in sql:
        tb2 = re.compile(r" join (.+?) (.+?) on", flags=re.I | re.S).findall(sql)
        if len(tb2) == 0:
            raise Exception('join表没有表别名，请检查')
    # 不应该有as
    if ' as ' in sql:
        raise Exception('表别名不需要as关键字，请删除')
    # 没有where的不需要SQL
    if ' where ' not in sql:
        raise Exception('没有where的sql用pandas就好了，不需要SQL这么折腾')
    # 判断是否有大写字母
    for s in sql:
        if s.isupper():
            raise Exception('SQL应该是全小写的，但是你传进来的SQL出现了大写字母')
    # 检查字段名是否带表别名
    sql_p = list(sqlparse.parse(sql)[0].flatten())
    for i in range(1, len(sql_p), 1):
        first = sql_p[i - 1]
        second = sql_p[i]
        if second._get_repr_name()=='Name' and second.value.strip() in table_1_cols and first.value!='.':
            raise Exception('字段名没有表别名，请检查，写法应该是：table_1_别名.%s'%second.value.strip())
        if second._get_repr_name()=='Name' and second.value.strip() in table_2_cols and first.value!='.':
            raise Exception('字段名没有表别名，请检查，写法应该是：table_2_别名.%s' % second.value.strip())
    return

def get_all_columns(sql):
    """解析获取所有涉及的字段名"""
    sql_p = list(sqlparse.parse(sql)[0].flatten())
    all_cols = []
    for i in range(2, len(sql_p), 1):
        first = sql_p[i - 2]
        second = sql_p[i - 1]
        third = sql_p[i]
        if first._get_repr_name() == 'Name' \
                and second._get_repr_name() == 'Punctuation' \
                and second.value == '.' \
                and third._get_repr_name() == 'Name':
            all_cols.append('%s.%s' % (first.value.strip(), third.value.strip()))
    return set(all_cols)

def tb_condition_cols_parser(sql):
    """
    从SQL中提取表名，别名，条件，涉及到的字段.
    locals是上层模块的globals，包含所有的对象，是字典类型
    """
    # 去掉换行符,多个空格只保留1个
    sql = re.compile(r'\s{2,100}', flags=re.I | re.S).subn(' ', sql.replace('\n', ' '))[0].strip()
    # 取update的表名和表别名
    tb1 = re.compile(r"update (.+?) (.+?)set", flags=re.I | re.S).findall(sql)[0]
    tb1, tb1_a = tb1[0].strip(), tb1[1].strip()
    # 提取join的表名和表别名
    if ' left join ' in sql:
        tb2 = re.compile(r"left join (.+?) (.+?)on", flags=re.I | re.S).findall(sql)[0]
        tb2, tb2_a = tb2[0].strip(), tb2[1].strip()
        join_type = 'left'
    elif ' join ' in sql:
        tb2 = re.compile(r"join (.+?) (.+?)on", flags=re.I | re.S).findall(sql)[0]
        tb2, tb2_a = tb2[0].strip(), tb2[1].strip()
        join_type = 'inner'
    else:
        tb2, tb2_a = '', ''
        join_type = ''
    # 提取set的内容
    if ' join ' not in sql:
        set_sql = re.compile(r" set(.+?) where ", flags=re.I | re.S).findall(sql)[0].strip()
    elif ' left join ' in sql:
        set_sql = re.compile(r" set(.+?) left join ", flags=re.I | re.S).findall(sql)[0].strip()
    elif ' join ' in sql:
        set_sql = re.compile(r" set(.+?) join ", flags=re.I | re.S).findall(sql)[0].strip()
    else:
        set_sql = ''
    # on 内容
    if ' join ' in sql:
        on_sql = re.compile(r" on(.+?) where ", flags=re.I | re.S).findall(sql)[0].strip()
    else:
        on_sql = ''
    # 提取where条件
    where_sql = re.compile(r" where(.*)", flags=re.I | re.S).findall(sql)[0].strip()
    # list(sqlparse.parse(where_sql)[0].get_sublists())
    # 提取a表涉及的所有列
    all_cols = get_all_columns(sql)
    tb1_cols = [col for col in all_cols if col.startswith(tb1_a + '.')]
    tb2_cols = [col for col in all_cols if col.startswith(tb2_a + '.')]
    # 组装成一个字典返回
    sql_info = {'tb1': tb1, 'tb1_a': tb1_a,
                'tb2': tb2, 'tb2_a': tb2_a,
                'join_type': join_type,
                'on_sql': on_sql,
                'set_sql': set_sql, 'where_sql': where_sql,
                'tb1_cols': tb1_cols, 'tb2_cols': tb2_cols}
    return sql_info

def tb1_join_tb2_parser(sql_info):
    """将涉及到的tb2字段全部join到tb1中，用左联接"""
    # 如果没有table2，返回空
    if not sql_info['tb2']:
        return {}
    # 取出带别名的tb1,tb2字段名
    tb1_cols = sql_info['tb1_cols']
    tb2_cols = sql_info['tb2_cols']

    # list的任意一个元素是否在string中的函数
    def list_in_string(string, list):
        for item in list:
            if item in string:
                return item

    # 找到两个表的关联字段
    left_on = []
    right_on = []
    for on_sql_sub in sql_info['on_sql'].split(' and '):
        # item=sql_info['on_sql'].split(' and ')[0]
        left_on.append(list_in_string(on_sql_sub, tb1_cols))
        right_on.append(list_in_string(on_sql_sub, tb2_cols))
    join_parser = {
        'left_on': left_on,
        'right_on': right_on
    }
    return join_parser

def tb1_filter_and_update_parser(sql_info):
    """
    前面已经将tb2 join 到 tb1 中，下面就是执行 where 和 set 操作了
    """
    # 取出set和where语句
    set_sql = sql_info['set_sql']
    where_sql = sql_info['where_sql']
    # 接下来就是转换 where_sql，使之符合pandas的语法
    where_sql_t = ''
    sub = ''
    for item in sqlparse.parse(where_sql)[0].flatten():
        item = item.value
        # 一个等号改2个等号
        if item == '=':
            sub += '=='
        elif item not in ('and', 'or'):  # 如果是断句，就应该小心了
            sub += item
        elif item in ('and', 'or'):
            # 处理and or断句
            if ' not in ' in sub:
                sub = '-' + sub
            else:
                sub = sub
            where_sql_t += sub
            where_sql_t += item
            sub = ''
    # 最后一个也放进去然后转回去
    if ' not in ' in sub:
        sub = ' -' + sub
    where_sql_t += sub
    where_sql = where_sql_t
    # in和not in，注意替换的顺序
    where_sql = where_sql.replace(' not in ', '.isin').replace(' in ', '.isin')
    # 在isin外面在嵌套一层括号
    where_sql_t = ''
    isin = False
    left_kh = False
    right_kh = False
    for item in sqlparse.parse(where_sql)[0].flatten():
        item = item.value
        if item == 'isin':
            isin = True
        if item == '(' and isin == True:
            item = '(('
            left_kh = True  # 已经提取左括号，标记
        elif item == ')' and isin == True and left_kh == True:
            item = '))'
            isin = False  # 将isin标记
            left_kh = False  # 将左括号标记
        where_sql_t += item
    # and or 转 & |
    where_sql_t = where_sql_t.replace(' and ', ') & (').replace(' or ', ') | (')
    where_sql = '(' + where_sql_t + ')'
    # null 和 notnull
    if ' is null ' in where_sql:
        where_sql = where_sql.replace(' is null ', '.isnull()')
    if ' is not null ' in where_sql:
        where_sql = where_sql.replace(' is not null ', '.notnull()')
    # <> 转 !=
    where_sql_t = ''
    for item in sqlparse.parse(where_sql)[0].flatten():
        if item._get_repr_name() == 'Comparison' and item.value.strip() == '<>':
            where_sql_t += '!='
        else:
            where_sql_t += item.value.strip()
    where_sql = where_sql_t
    # 将 a.id 写成 df['id'] 的格式
    tb1_cols = sql_info['tb1_cols']  # tb1涉及到的字段名，包含表别名
    tb2_cols = sql_info['tb2_cols']
    tb1, tb1_a = sql_info['tb1'], sql_info['tb1_a']
    for col in tb1_cols:
        col1 = col  # 此时已经包含表别名和字段名
        col2 = "%s['%s']" % (tb1, col.split('.')[-1])
        where_sql = where_sql.replace(col1, col2)
    # 需要注意的是，tb2的字段名需要增加 t_tmp_ 前缀标志
    # 注意这里是tb1，因为tb2已经join到tb1了
    for col in tb2_cols:
        col1 = col  # 此时已经包含表别名和字段名
        col2 = "%s['b_tmp_%s']" % (tb1, col.split('.')[-1])
        where_sql = where_sql.replace(col1, col2)
    # 对于set_sql要麻烦一些，要先分割，然后在替换
    # set_sql='a.age=b.b_tmp_age2+2, a.score=func(a.age, b.b_tmp_level2)'
    # set_sql = 'a.age=b.b_tmp_age2+2'
    set_sql_t = sqlparse.parse(set_sql)[0].tokens[0]
    if set_sql_t._get_repr_name() == 'Comparison':
        set_sql = [set_sql_t.value]
    if set_sql_t._get_repr_name() == 'IdentifierList':
        set_sql = [t.value for t in set_sql_t.get_sublists()]
    # 表名替换
    set_sqls2 = []
    for set_t in set_sql:
        for col in tb1_cols:
            col1 = col
            col2 = "%s.loc[wz, '%s']" % (tb1, col.split('.')[-1])
            set_t = set_t.replace(col1, col2)
        for col in tb2_cols:
            col1 = col
            # 注意，这里tb用tb1，col是tb2的，而且要加上 b_tmp_ 前缀
            col2 = "%s.loc[wz, 'b_tmp_%s']" % (tb1, col.split('.')[-1])
            set_t = set_t.replace(col1, col2)
        set_sqls2.append(set_t)
    set_sqls = set_sqls2
    #
    update_parser = {
        'where_sql': where_sql,
        'set_sqls': set_sqls  # 是一个列表
    }
    return update_parser

def update_tb2_column_name_then_join(sql_info, join_parser, table_1, table_2):
    """更新B表字段名, 然后A表关联B表，对关联后的结果校验"""
    # 重命名B表所有涉及的列
    tb2_cols_rename = []  # 其他涉及到的，不是关联key字段也要重命名
    for col in sql_info['tb2_cols']:
        col1 = col.split('.')[-1]
        col2 = 'b_tmp_' + col1
        if col1 not in table_2.columns:
            raise Exception('在%s中找不到%s字段' % (sql_info['tb2'], col1))
        table_2.rename(columns={col1: col2}, inplace=True)
        tb2_cols_rename.append(col2)
    # 重命名join on的列
    tb1_join_col = [col.split('.')[-1] for col in join_parser['left_on']]  # tb1去掉表名
    tb2_join_col = ['b_tmp_%s' % col.split('.')[-1] for col in join_parser['right_on']]  # tb2去表名再加上tmp标志
    # 关联
    len1 = len(table_1)
    table_1 = pd.merge(left=table_1,
                       right=table_2[tb2_cols_rename],
                       left_on=tb1_join_col,
                       right_on=tb2_join_col,
                       how=sql_info['join_type'])
    # 检查前后数据量变化
    len2 = len(table_1)
    print('注意，关联后数据量变化 %d --> %d' % (len1, len2)) if len1 != len2 else None
    # 检查数据类型的变化
    for col in [col for col in table_2.columns if col.startswith('b_tmp_')]:
        dtype_new = table_1.dtypes[col]
        dtype_old = table_2.dtypes[col]
        if dtype_old != dtype_new:
            print('注意，左右表关联后，原来的列 %s 的数据类型 %s -- > %s' % (col, str(dtype_old), str(dtype_new)))
    # 重命名B表所有涉及的列回原来的列名
    for col in tb2_cols_rename:
        col1 = col
        col2 = col.replace('b_tmp_', '')
        table_2.rename(columns={col1: col2}, inplace=True)
    # 最后返回
    return table_1

def update_tb1(sql_info, update_parser, table_1, func1, func2, func3):
    """根据update_parser更新table1"""
    exec('wz = ' + update_parser['where_sql'])
    for set_sql in update_parser['set_sqls']:
        exec(set_sql)
    return table_1

def delete_tb1_that_tmp_column(table_1):
    # 删除以b_tmp_开头的临时列
    for col in table_1.columns:
        if col.startswith('b_tmp_'):
            del table_1[col]
    return table_1

def sql_update_pd(sql, table_1=None, table_2=None, func1=None, func2=None, func3=None):
    """主函数"""
    # 第一步，判断SQL是否正确
    table_1_cols = table_1.columns.tolist()
    table_2_cols = table_2.columns.tolist() if isinstance(table_2, pd.DataFrame) else []
    judge_format(sql, table_1_cols, table_2_cols)
    # 第二步，解析SQL，同时再次判断SQL语法是否正确
    sql_info = tb_condition_cols_parser(sql)
    # 第三步，join关联解析
    join_parser = tb1_join_tb2_parser(sql_info)
    # 第四步，update解析
    update_parser = tb1_filter_and_update_parser(sql_info)
    # 第五步，更新B表的字段名
    # 第六步，A join B
    # 第七步，将B表的列名改回原来的
    if sql_info['tb2']:
        table_1 = update_tb2_column_name_then_join(sql_info, join_parser, table_1=table_1, table_2=table_2)
    # 第八步，update A
    table_1 = update_tb1(sql_info, update_parser, table_1 = table_1, func1=func1, func2=func2, func3=func3)
    # 第9步，删除A中临时列
    table_1 = delete_tb1_that_tmp_column(table_1 = table_1)
    # 第10步，返回A
    return table_1

def test():
    # 构造测试数据
    df1 = pd.DataFrame(data={'id': [1, 2, 3, 4], 'age': [2, 3, 4, 5], 'score': [3, 4, 7, 6], 'heigh': [5, 6, 8, 7]})
    df2 = pd.DataFrame(data={'id': [1, 2, 3, 4], 'age2': [3, 4, 5, 6], 'score2': [5, 6, 7, 7]})
    print(df1)
    print(df2)

    # 测试语法是否正确
    sql = """ update table_1 set age=id+1 """  # 没有表别名
    sql = """ update table_1 as a set age=id+1 """  # 不应该用as
    sql = """ update table_1 a set a.age = a.id + a.score"""  # 没有where条件，就没必要搞SQL这么麻烦了
    judge_format(sql)

    # 正常是写法，字段名也应该带表别名
    sql = """ update table_1 a set a.age = a.id + a.score where a.heigh>=5"""
    judge_format(sql)

    # ------------------------------------------------------------
    # 单表更新
    sql = "update table_1 a set a.age=a.id*2 where a.id>=3"
    sql_update_pd(sql, table_1 = df1)
    # ------------------------------------------------------------
    # 联合更新
    sql = "update table_1 a " \
          "set a.age=a.id+b.age2+1 " \
          "left join table_2 b " \
          "on a.id=b.id " \
          "where a.age>2 and b.age2<6"  # 更新中间两行
    new_df_1 = sql_update_pd(sql, table_1=df1, table_2=df2)

    # ------------------------------------------------------------
    # 使用自定义python函数
    def add(a, b):
        return a * b
    def add2(a,b):
        return a-b

    df1['t']=0
    sql = "update table_1 a set a.score=add(a.id, a.age), a.t = add2(a.age, a.heigh) where a.id>=3"
    new_df_1 = sql_update_pd(sql, table_1=df1, func1=add, func2=add2)

    # ------------------------------------------------------------
    # 多字段更新
    sql = "update table_1 a set a.score=a.id+2, a.heigh=a.age+2 where a.id>=3"
    new_df_1 = sql_update_pd(sql, table_1=df1)


import os

import torch
import pandas as pd
from transformers import BertTokenizer, BertForSequenceClassification, AdamW, TrainingArguments, Trainer
from torch.utils.data import DataLoader, TensorDataset, random_split

from transformers import BertModel
import torch.nn as nn

import wandb

class BertRegression(nn.Module):
    def __init__(self, num_outputs):
        super(BertRegression, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.regressor = nn.Linear(self.bert.config.hidden_size, num_outputs)

    def forward(self, input_ids, attention_mask):
        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
        pooled_output = outputs[1]
        return self.regressor(pooled_output)

use_wandb = True

# 1. Data Preparation
data = pd.read_csv('training.csv')
tweets = data['SocialMediaFeed'].tolist()
labels = data[['NVDA', 'ING', 'SAN', 'PFE', 'CSCO']].values.tolist()

device = 'cuda' if torch.cuda.is_available() else 'cpu'

# 2. Preprocess the Data
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
encoded_data = tokenizer(tweets, truncation=True, padding=True, max_length=256, return_tensors="pt")
input_ids = encoded_data['input_ids'].to(device)
attention_masks = encoded_data['attention_mask'].to(device)
labels = torch.tensor(labels).to(device)
labels *= 100

dataset = TensorDataset(input_ids, attention_masks, labels)
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])

train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=8)
val_dataloader = DataLoader(val_dataset, batch_size=8)

# 3. Model Setup
model = BertRegression(num_outputs=5).to(device)

# 4. Training
optimizer = AdamW(model.parameters(), lr=1e-5)
loss_fn = torch.nn.MSELoss()

if use_wandb:
    wandb.init(project="stock-prediction")
    wandb.watch(model)

MODEL_DIR = 'model_checkpoints'
if not os.path.exists(MODEL_DIR):
    os.makedirs(MODEL_DIR)

epochs = 10
for epoch in range(epochs):
    model.train()
    for batch in train_dataloader:
        input_ids, attention_mask, batch_labels = batch

        outputs = model(input_ids, attention_mask=attention_mask)
        loss = loss_fn(outputs, batch_labels)
        print("Epoch:", str(epoch), "-- Training Loss:", str(loss.item()))
        if use_wandb:
            wandb.log({"Training Loss": loss.item()})

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    # Validation
    model.eval()
    val_loss = 0
    with torch.no_grad():
        for batch in val_dataloader:
            input_ids, attention_mask, batch_labels = batch

            outputs = model(input_ids, attention_mask=attention_mask)
            loss = loss_fn(outputs, batch_labels)
            val_loss += loss.item()
            print("Epoch:", str(epoch), "-- Validation Loss:", str(loss.item()))

    print(f"Epoch: {epoch+1}, Validation Loss: {val_loss/len(val_dataloader)}")
    if use_wandb:
        wandb.log({"Validation Loss": val_loss/len(val_dataloader)})

    model_checkpoint_path = os.path.join(MODEL_DIR, f'epoch_{epoch+1}.pth')
    torch.save(model.state_dict(), model_checkpoint_path)
    if use_wandb:
        wandb.save(model_checkpoint_path)


import csv
import os
from typing import Dict, Generator, Iterable, Optional, Tuple

import numpy as np

from ..classification import ModuleClassifier
from ..classification.settings import CLASS_FIELD, TEXT_FIELDS

class Preprocessor:
    def __init__(self, csv_file: str):
        self.csv_file = csv_file

    def read_csv(self) -> Iterable[Dict[str, str]]:
        with open(self.csv_file) as f:
            reader = csv.DictReader(f, dialect="excel")
            for row in reader:
                if row:
                    yield row

    def write_fasttext(
        self,
        target_file: str,
        split: Tuple[float, float, float] = (1.0, 0.0, 0.0),
        suffixes=(".train", ".dev", ".test"),
        seed: Optional[int] = None,
    ):
        if not sum(split) == 1.0:
            raise ValueError("Split must sum up to 1.0")
        np.random.seed(seed)

        target_files = [target_file + suffix for suffix in suffixes]
        handlers = [open(f, "w") for f in target_files]

        for line in self.generate_fasttext_lines():
            handler = np.random.choice(handlers, p=split)
            handler.write(line)
            handler.write(os.linesep)

        for handler in handlers:
            handler.close()

    def generate_fasttext_lines(self) -> Generator[str, None, None]:
        return (
            ModuleClassifier.fasttext_line(row, TEXT_FIELDS)
            for row in self.read_csv()
            if row.get(CLASS_FIELD)
        )


class Student:
    def __init__(self, name, surname, gender):
        self.name = name
        self.surname = surname
        self.gender = gender
        self.finished_courses = []
        self.courses_in_progress = []
        self.grades = {}

    def add_courses(self, course_name):
        self.finished_course.append(course_name)

    def rate_lecturer(self, lecturer, course, grade):
        if isinstance(lecturer,
                      Lecturer) and course in self.courses_in_progress and course in lecturer.courses_attached:
            if course in lecturer.grades:
                lecturer.grades[course] += [grade]
            else:
                lecturer.grades[course] = [grade]
        else:
            return 'Ошибка'

    def sr_bal(self):
        all_balls = 0
        sum_all_balls = 0
        for keys in self.grades.values():
            for key_items in keys:
                all_balls += 1
                sum_all_balls += key_items
        if all_balls != 0:
            return sum_all_balls / all_balls
        else:
            return 'Заполните оценки'

    def __str__(self):
        return f'Имя: {str(self.name)} \nФамилия: {self.surname} \nСредняя оценка за домашние задания: {self.sr_bal()} \nКурсы в процессе изучения:{self.courses_in_progress} \nЗавершенные курсы: Введение в программирование'

    def __lt__(self, other):
        stud1 = self.sr_bal()
        stud2 = other.sr_bal()
        return stud1 < stud2
    def __gt__(self, other):
        stud1 = self.sr_bal()
        stud2 = other.sr_bal()
        return stud1 > stud2

class Mentor:
    def __init__(self, name, surname):
        self.name = name
        self.surname = surname
        self.courses_attached = []

class Lecturer(Mentor):
    def __init__(self, name, surname):
        super().__init__(name, surname)
        self.grades = {}

    def __str__(self):
        print(f'Имя: {self.name}')
        print(f'Фамилия: {self.surname}')
        print(f'Средняя оценка за лекции: {self.sr_bal}')

    def sr_bal(self):
        all_balls = 0
        sum_all_balls = 0
        for keys in self.grades.values():
            for key_items in keys:
                all_balls += 1
                sum_all_balls += key_items
        if all_balls != 0:
            return sum_all_balls / all_balls
        else:
            return 'Заполните оценки'

    def __str__(self):
        return f'Имя: {self.name} \nФамилия: {self.surname} \nСредняя оценка за лекции: {self.sr_bal()}'

    def __lt__(self, other):
        lector1 = self.sr_bal()
        lector2 = other.sr_bal()
        return lector1 < lector2
    def __gt__(self, other):
        lector1 = self.sr_bal()
        lector2 = other.sr_bal()
        return lector1 > lector2

class Reviewer(Mentor):
    def __init__(self, name, surname):
        super().__init__(name, surname)

    def rate_hw(self, student, course, grade):
        if isinstance(student, Student) and course in self.courses_attached and course in student.courses_in_progress:
            if course in student.grades:
                student.grades[course] += [grade]
            else:
                student.grades[course] = [grade]
        else:
            return 'Ошибка'

    def __str__(self):
        return f'Имя: {self.name} \nФамилия: {self.surname}'

#Студенты
stud1 = Student('Ruoy', 'Eman', 'female')
stud1.courses_in_progress += ['Python']
stud1.courses_in_progress += ['Git']
stud1.courses_in_progress += ['Engish']
# stud1.grades['Python'] = [5, 5, 3, 2, 5]
# stud1.grades['Git'] = [3, 4, 2, 5, 3]
# stud1.grades['Engish'] = [3, 4, 2, 5, 3]
stud1.finished_courses += ['php']
stud1.finished_courses += ['c++']
stud1.finished_courses += ['c#']

stud2 = Student('Ivan', 'Eman', 'male')
stud2.courses_in_progress += ['Python2']
stud2.courses_in_progress += ['Git2']
stud2.courses_in_progress += ['Engish2']
# stud2.grades['Python2'] = [5, 5, 3, 2, 5]
# stud2.grades['Git2'] = [3, 4, 2, 5, 3]
# stud2.grades['Engish2'] = [3, 4, 2, 5, 3]
stud2.finished_courses += ['php2']
stud2.finished_courses += ['c++2']
stud2.finished_courses += ['c#2']

#Лекторы
lector1 = Lecturer('Alexandr', 'Ivanovich')
lector1.courses_attached += ['Python']
lector1.courses_attached += ['Git']
lector1.courses_attached += ['Git2']

lector2 = Lecturer('Ivan', 'Aleksandrovich')
lector2.courses_attached += ['Engish2']
lector2.courses_attached += ['c++2']
lector2.courses_attached += ['c#2']

#Ревьюеры
rev1 = Reviewer('Grigirii', 'Vladlenovich')
rev1.courses_attached += ['Python']
rev1.courses_attached += ['Git']
rev1.courses_attached += ['Git2']

rev2 = Reviewer('Sergey', 'Viktorovich')
rev2.courses_attached += ['Engish2']
rev2.courses_attached += ['c++2']
rev2.courses_attached += ['c#2']

#Начисление оценок
stud1.rate_lecturer(lector1 ,'Python', 5)
stud1.rate_lecturer(lector2 ,'Git', 5)
stud1.rate_lecturer(lector1 ,'Python', 3)
stud1.rate_lecturer(lector2 ,'Git', 4)

stud2.rate_lecturer(lector2 ,'Engish2', 5)
stud2.rate_lecturer(lector2 ,'Engish2', 4)
stud2.rate_lecturer(lector2 ,'Engish2', 3)
stud2.rate_lecturer(lector2 ,'Engish2', 5)

rev1.rate_hw(stud1,'Python', 5)
rev1.rate_hw(stud1,'Git', 5)
rev1.rate_hw(stud2,'Git2', 5)
rev1.rate_hw(stud1,'Python', 4)
rev1.rate_hw(stud1,'Git', 4)
rev1.rate_hw(stud2,'Git2', 3)
rev1.rate_hw(stud1,'Python', 3)
rev1.rate_hw(stud1,'Git', 3)
rev1.rate_hw(stud2,'Git2', 3)

rev2.rate_hw(stud2, 'English2', 4 )
rev2.rate_hw(stud2, 'English2', 5 )
rev2.rate_hw(stud2, 'English2', 3 )
rev2.rate_hw(stud2, 'English2', 4 )
rev2.rate_hw(stud2, 'English2', 2 )

rev2.rate_hw(stud1, 'English', 5 )
rev2.rate_hw(stud1, 'English', 4 )
rev2.rate_hw(stud1, 'English', 5 )
rev2.rate_hw(stud1, 'English', 4 )
rev2.rate_hw(stud1, 'English', 3 )

def sr_ball_po_stud(students_list, predmet):
    all_bals = 0
    sum_all_bals = 0
    for i in students_list:
        for cours in i.grades:
            if cours == predmet:
                for balls in i.grades[cours]:
                    all_bals += 1
                    sum_all_bals += balls

    if all_bals != 0:
        return f'Средний балл студентов по предмету {predmet}: {sum_all_bals / all_bals  }'
    else:
        return f'Такого предмета {predmet} нет у студентов'

def sr_ball_po_lectors(lectors_list, predmet):
    all_bals = 0
    sum_all_bals = 0
    for i in lectors_list:
        for cours in i.grades:
            if cours == predmet:
                for balls in i.grades[cours]:
                    all_bals += 1
                    sum_all_bals += balls

    if all_bals != 0:
        return f'Средний балл лекторов по предмету {predmet}: {sum_all_bals / all_bals  }'
    else:
        return f'Такого предмета {predmet} нет у лекторов'

print(lector1)
print()
print(lector2)
print()
print(stud1)
print()
print(stud2)
print()
print(lector1 > lector2)
print()
print(lector1 < lector2)
print()
print(stud1 > stud2)
print()
print(stud1 < stud2)
print()
print(sr_ball_po_stud([stud1, stud2], 'Python' ))
print()
print(sr_ball_po_stud([lector1, lector2], 'Python' ))


import numpy as np

class Solution:
    def maximalSquare(self, grid):
        height = len(grid)
        if height == 0:
            return 0
        width = len(grid[0])
        grid = np.array(grid)
        sub = lambda x, y, d: '0' not in grid[y:y+d, x:x+d]  # noqa: E731

        to_search = []
        for i in range(height):
            for j in range(width):
                if grid[i][j] == '1':
                    to_search.append([i, j])
        if len(to_search) == 0:
            return 0

        dim = 1
        found = True
        while found:
            found = False
            new_to_search = []
            for p in to_search:
                if p[0]+dim < height and p[1]+dim < width \
                        and sub(p[1], p[0], dim+1):
                    found = True
                    new_to_search.append(p)
            if found:
                dim += 1
            to_search = new_to_search

        return dim**2

LeetCode/30 Day LeetCoding Challenge/Week 4/27_maximal_square.py


import os
from openpyxl import load_workbook
import re
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import stopwords
from itertools import groupby

special_words = ['prosses', 'acsses']

class DataLoader():
    def __init__(self, wb, sheetname):
        self.sheetname = sheetname
        self.sheet = wb[self.sheetname]
        self.questions, self.categories, id_cate_dict = self._get_questions()

    def _get_questions(self):
        m_row = self.sheet.max_row
        id_cate_dict = []
        questions = []
        categories = []
        for i in range(2, m_row + 1):
            cell_obj = self.sheet.cell(row = i, column = 2)
            if not cell_obj.value:
                continue
            questions.append(cell_obj.value)
            categories.append(self.sheet.cell(row=i, column=6).value)
            try:
                id_cate_dict.append((self.sheet.cell(row=i, column=1).value.split('/')[-1].strip(), self.sheet.cell(row=i, column=6).value.strip().replace('\n', ' ')))
            except:
                print(i)
        return questions, categories, id_cate_dict

    def _check_pattern(self, token):
        method_patterns = [re.compile('\w+\.+'), re.compile('\w+\_+')]
        for method_pattern in method_patterns:
            if re.search(method_pattern, token):
                return '<method>'
            elif re.search('\_', token):
                return '<method>'
            else:
                return token

    def _process_ques(self):
        new_ques = []
        stop_words = set(stopwords.words('english'))
        for idx, line in enumerate(self.questions):
            if not line:
                continue
            terms = line.lower().split()
            for term_id, term in enumerate(terms):
                terms[term_id] = re.sub('[\"\(\)\`\=\:\[\]\-\,\?\+\/]*', ' ',term)
                if re.search(re.compile("(\w+)\'s"), terms[term_id]):
                    terms[term_id] = terms[term_id].split("'")[0]
                terms[term_id] = re.sub("[\']*", '', terms[term_id])
                terms[term_id] = re.sub(' +', '', terms[term_id])
                terms[term_id] = self._check_pattern(terms[term_id])
                terms[term_id] = WordNetLemmatizer().lemmatize(terms[term_id], 'v')
                if terms[term_id] not in special_words:
                    temp_token = WordNetLemmatizer().lemmatize(terms[term_id], 'n')
                    if terms[term_id] != temp_token and not re.search(r'ss$', terms[term_id]):
                        terms[term_id] = temp_token
                if terms[term_id] in stop_words or len(terms[term_id])==1:
                    terms[term_id] = ''
                if re.match(re.compile("(\d+)d$"), terms[term_id]):   # $ is the end of string
                    terms[term_id] = '<dim>'
                elif re.match(re.compile("(\d+)gb$"), terms[term_id]):   # $ is the end of string
                    terms[term_id] = '<mem>'
                elif terms[term_id].isdigit():
                    terms[term_id] = '<digit>'

            new_terms = [x[0] for x in groupby(terms)]
            new_terms = ' '.join(new_terms)
            new_terms = re.sub(r"[^0-9a-z,.?!<>' ]", ' ', new_terms)
            new_terms = re.sub(' +', ' ', new_terms)
            category = self.categories[idx].strip().replace('\n', ' ')
            new_ques.append(new_terms.strip()+'\t'+category+'\n')
        return new_ques

def save_ques(data, fn):
    fw = open(os.path.join('..', 'data', fn), 'w')
    for item in data:
        try:
            fw.write(item)
        except TypeError:
            print(item)
            continue
    fw.close()
    print('%s has finished writing...'%fn)

# wb = load_workbook(filename=os.path.join('..', 'data', 'dl_framework_stackoverflow_questions.xlsx'))
# tf_sheet = wb['TensorFlow(sample)']
# pt_sheet = wb['PyTorch(sample)']
# dj_sheet = wb['DeepLearning4J(sample)']
#
# tf_questions = DataLoader(wb, 'TensorFlow(sample)')._process_ques()
# pt_questions = DataLoader(wb, 'PyTorch(sample)')._process_ques()
# dj_questions = DataLoader(wb, 'DeepLearning4J(sample)')._process_ques()
#
# save_ques(tf_questions, 'tensforflow.txt')
# save_ques(dj_questions, 'dl4j.txt')
# save_ques(pt_questions, 'pytorch.txt')
# save_ques(tf_questions+pt_questions+dj_questions, 'total.txt')


from admin import*
from Student import Student
from LL import *
from que import *

class Home:
    def __init__(self, studentinfo, qued):
        self.oto = None
        condition =True
        while (condition):
            print("\n\t\t\t\t\t\t\t=========____STUDENT PORTAL____=========\n")
            num=int(input("Press 1 to Enter Admin Panel \nPress 2 To Enter Student Portal \nPress 3 to EXIT "))
            if num == 1:
                self.oto="admin"
                condition=False
            elif num == 2:
                self.oto="student"
                condition=False
            elif num == 3:
                condition = False
            else:
                print("Invalid Entry \n")
        if (self.oto == "admin"):
            admin(studentinfo,qued,Home)
        elif (self.oto == "student"):
            Student(studentinfo,qued,Home)
        else:
            print("=++= GOOD BYE! =++=")

def main():
    studentinfo = LL()
    q = que(5)
    Home(studentinfo,q)

if __name__ == "__main__":
    main()


import cv2
import os
import numpy as np

mate_color_dict = {"blue_button": [[50, 110], [150, 200], [220, 255]],
                   "white": [[240, 255], [240, 255], [240, 255]],
                   "grey": [[200, 230], [200, 230], [200, 230]],
                   "red": [[200, 255], [40, 80], [40, 80]],
                   "blue": [[90, 110], [180, 210], [230, 255]],
                   }

def is_contain(rects):
    rects = [[rect[0], rect[1], rect[0] + rect[2], rect[1] + rect[3]] for rect in rects]
    return (rects[0][0] < rects[1][0] and rects[0][1] < rects[1][1] and rects[1][2] < rects[0][2]
            and rects[1][3] < rects[0][3]) or \
           (rects[1][0] < rects[0][0] and rects[1][1] < rects[0][1] and rects[0][2] < rects[1][2]
            and rects[0][3] < rects[1][3])

def is_color(pixel, thresh):
    return thresh[2][1] >= pixel[0] >= thresh[2][0] and thresh[1][1] >= pixel[1] >= thresh[1][0] and \
           thresh[0][1] >= pixel[2] >= thresh[0][0]

def binary_thresh_pic(img, color_keys):
    height, width, channel = img.shape
    bg = np.zeros((height, width), dtype=np.uint8)
    for h in range(height):
        for w in range(width):
            for i, color_key in enumerate(color_keys):
                if is_color(img[h, w, :], mate_color_dict[color_key]):
                    bg[h, w] = 255
                    break
                if i == len(color_keys) - 1:
                    bg[h, w] = 0
    return bg

img = cv2.imread("12573.jpg")
# img = cv2.resize(img, None, None, 0.5, 0.5)
h, w, c = img.shape

star_h = h // 4
clip_h = h // 3
star_w = w * 9 // 10
clip_w = w * 3 // 5

img = img[star_h: star_h + clip_h, star_w:, :]

# img = cv2.resize(img, None, None, 0.5, 0.5)

# h, w, c = img.shape

binary_thresh = binary_thresh_pic(img, ["red", "blue"])

kernel = np.ones((3, 3), np.uint8)
dilate = cv2.dilate(binary_thresh, kernel)

cv2.imshow("thresh", dilate)

image, contours, hierarchy = cv2.findContours(dilate, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

rects = [cv2.boundingRect(box) for box in contours]

for rect in rects:
    cv2.rectangle(img, (rect[0], rect[1]), (rect[0] + rect[2], rect[1] + rect[3]), (0, 0, 255), 1)

rects = sorted(rects, key=lambda x: x[0])

if len(rects) == 2 and rects[0][0] < w / 6 and rects[0][1] > h * 2 / 3 and rects[1][0] > w / 2 and rects[1][1] > h / 2 \
        and rects[0][2] > w / 4:
    print("has_finish")

cv2.imshow("img", img)
cv2.waitKey(0)


#!/usr/bin/env python3

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import itertools
import open3d
import subprocess as sp
import shlex
import copy
import scipy

def set_rotation(g, azimuth=0, tilt=0):
    """rotates a Trimesh around its bounding box center. In addition to setting
    azimuth and tilt, the mesh is rotated so that elevation (data z-axis) is
    oriented up-down (display y-axis).

    Args:
        g (Trimesh): Open3D Trimesh
        azimuth (float): viewer azimuth (0=N, 90=W, 180=S, 270=E)
        tilt (float): tilt the mesh towards viewer
    Returns:
    """
    ctr = g.get_axis_aligned_bounding_box().get_center()
    r1 = open3d.geometry.get_rotation_matrix_from_zyx(np.array([0 , -azimuth*np.pi/180., -np.pi/2]))
    r2 = open3d.geometry.get_rotation_matrix_from_zyx(np.array([0, 0, tilt*np.pi/180.]))
    g.rotate(r1, center=ctr).rotate(r2, center=ctr)

def dump_mp4(mov, output_filename='movie.mp4', fps=60, crf=18):
    """dump frames to mp4

    adapted from https://stackoverflow.com/a/61281547/6474403

    Args:
        mov (ndarray): array of movie frames (num_frames, height, width, 3) uint8
        output_filename (str): output filename (will overwrite)
        fps (float): frame per second
        crf (float): constant rate factor, 0-51 with 0 being lossless, and 51 being worst
    """
    if mov is None:
        return

    width, height, n_frames= mov.shape[2], mov.shape[1], len(mov)

    # Open ffmpeg application as sub-process
    # FFmpeg input PIPE: RAW images in BGR color format
    # FFmpeg output MP4 file encoded with HEVC codec.
    # Arguments list:
    # -y                   Overwrite output file without asking
    # -s {width}x{height}  Input resolution width x height (1344x756)
    # -pixel_format bgr24  Input frame color format is BGR with 8 bits per color component
    # -f rawvideo          Input format: raw video
    # -r {fps}             Frame rate: fps (25fps)
    # -i pipe:             ffmpeg input is a PIPE
    # -vcodec libx265      Video codec: H.265 (HEVC)
    # -pix_fmt yuv420p     Output video color space YUV420 (saving space compared to YUV444)
    # -crf 24              Constant quality encoding (lower value for higher quality and larger output file).
    # {output_filename}    Output file name: output_filename (output.mp4)
    process = sp.Popen(shlex.split(f'ffmpeg -y -s {width}x{height} -pixel_format bgr24 -f rawvideo -r {fps} -i pipe: -vcodec libx265 -pix_fmt yuv420p -crf {crf} {output_filename}'), stdin=sp.PIPE)

    # Build synthetic video frames and write them to ffmpeg input stream.
    for i in range(n_frames):
        # Build synthetic image for testing ("render" a video frame).
        # img = np.full((height, width, 3), 60, np.uint8)
        # cv2.putText(img, str(i+1), (width//2-100*len(str(i+1)), height//2+100), cv2.FONT_HERSHEY_DUPLEX, 10, (255, 30, 30), 20)  # Blue number
        #img = cv2.cvtColor(mov[i], cv2.COLOR_RGB2BGR)
        img = mov[i][:, :, [2,1,0]]

        # Write raw video frame to input stream of ffmpeg sub-process.
        process.stdin.write(img.tobytes())

    # Close and flush stdin
    process.stdin.close()

    # Wait for sub-process to finish
    process.wait()

    # Terminate the sub-process
    process.terminate()  # Note: We don't have to terminate the sub-process (after process.wait(), the sub-process is supposed to be closed).

def add_bbox_points(mesh):
    """Add bounding box corners as vertices of a Trimesh. This prevents
    translation drift due to re-centering after decimation. The added
    vertices are not part of any triangles so they are preserved during
    decimation.
    """
    c = copy.deepcopy(mesh)
    xyz = np.vstack((np.array(c.vertices), np.array(c.get_axis_aligned_bounding_box().get_box_points())))
    c.vertices = open3d.utility.Vector3dVector(xyz)
    return c

def pimp_meshes(meshes, **kwargs):
    """Staging a list of meshes for visualization. Vertices are colored, and
    meshes are re-oriented and re-aligned.

    Args:
        meshes (list): list of TriMesh objects
        colormap (): Matplotlib colormap or an RGB integer 3-tuple (0-255)
        azimuth (float): initial orientation (see `set_rotation()`)
        tilt (float): initial orientation (see `set_rotation()`)
    Returns:
        meshes (list): new list of TriMesh objects (input are unchanged)
    """
    defaults = dict(
        colormap=plt.cm.winter,
        azimuth=0,
        tilt=0,
    )
    colormap = kwargs.get('colormap', defaults['colormap'])
    azimuth = kwargs.get('azimuth', defaults['azimuth'])
    tilt = kwargs.get('tilt', defaults['tilt'])

    meshes = [copy.deepcopy(x) for x in meshes]
    # set vertex colors based on elevation
    for x in meshes:
        zvec = np.array(x.vertices)[:, 2]
        zvec = (zvec-np.min(zvec))/(np.max(zvec)-np.min(zvec))
        if isinstance(colormap, matplotlib.colors.LinearSegmentedColormap):
            x.vertex_colors = open3d.utility.Vector3dVector(colormap(zvec)[:, 0:3])
        elif isinstance(colormap, list):
            x.paint_uniform_color(np.asarray(colormap)/255)
        x.compute_vertex_normals() # so lighting works
    # Set initial rotation and align the bbox ctrs
    ref = np.array(meshes[0].get_axis_aligned_bounding_box().get_center())
    for x in meshes:
        set_rotation(x, azimuth, tilt)
        x.translate(ref-np.array(x.get_axis_aligned_bounding_box().get_center()))

    return meshes

def twisty_viz(meshes, **kwargs):
    """Mesh surface visualization that cycles through a list of geometries
    while twisting back and forth.

    The UI allows lots of fiddling (changing view options, zoom, resize, etc.)
    and pressing `h` will print a list of keyboard shortcuts. Extra options
    added here include
    - press spacebar to pause/resume the animation
    - press c to capture the animation frame sequence, which is then returned
    as `mov` when the viewer is closed.

    NOTE LineSet for wireframe looks junky (cannot make the lines thicker)
    TODO relationship between change_field_of_view and the result??

    Args:
        meshes (list): list of `open3d.geometry.Geometry`s
        **kwargs (dict): some visualization options
    Returns:
        mov (ndarray): rgb frames (num_frames, height, width, 3) np.uint8
    """
    defaults = dict(
        bg_color=[255, 255, 255],
        twist_axis='x',
        twist_angle=10,
        num_frames=500,
        pause_factor=4,
        viz_height=640,
        viz_width=640
    )
    bg_color = kwargs.get('bg_color', defaults['bg_color'])
    twist_axis = kwargs.get('twist_axis', defaults['twist_axis'])
    twist_angle = kwargs.get('twist_angle', defaults['twist_angle'])
    num_frames = kwargs.get('num_frames', defaults['num_frames'])
    pause_factor = kwargs.get('pause_factor', defaults['pause_factor'])
    viz_height = kwargs.get('viz_height', defaults['viz_height'])
    viz_width = kwargs.get('viz_width', defaults['viz_width'])

    assert num_frames%4 == 0

    # Set up the visualizer
    class KeyPressManager():
        """Visualizer callbacks for keypress events"""
        def __init__(self):
            self.capture = False
            self.pause = False
        def key_callback_capture(self, vis, action, mods):
            """pressing `c` triggers capturing the next animation sweep"""
            if action == 1:
                print("# capturing the next animation sweep!")
                self.capture = True
        def key_callback_pause(self, vis, action, mods):
            """pressing `space` pauses/resumes the animation"""
            if action == 1:
                self.pause = not self.pause

    viz = open3d.visualization.VisualizerWithKeyCallback()
    kpm = KeyPressManager()
    viz.register_key_action_callback(32, kpm.key_callback_pause)
    viz.register_key_action_callback(67, kpm.key_callback_capture)
    viz.create_window(width=viz_width, height=viz_height, left=50, top=50, visible=True)
    ro = viz.get_render_option()
    ro.light_on = True
    ro.mesh_show_wireframe = True
    ro.mesh_show_back_face = True
    ro.background_color = np.array(bg_color)/255.

    # Set FOV
    viz.add_geometry(meshes[0])
    viz.get_view_control().change_field_of_view(-5)
    viz.clear_geometries()

    # Set up the mesh and twist animation sequences
    # angular: twist left-right-right-left (returns to the start)
    step_size =twist_angle/(num_frames/2)/(0.003*180/np.pi)  # 0.003 rad/pixel
    seq = [-step_size]*int(num_frames/4) + [step_size]*int(num_frames/2) + [-step_size]*int(num_frames/4)
    # geometries: pause(first geometry)-forward-pause(last geometry)-reverse
    stride = int((num_frames/2)//(len(meshes)+pause_factor))
    pad = int((num_frames/2)%(len(meshes)+pause_factor))
    num_pause = stride*pause_factor+pad
    x = list(itertools.chain(*[[i]*stride for i in range(len(meshes))]))
    meshes_viz = [0]*num_pause+x+[len(meshes)-1]*num_pause+x[::-1]
    meshes_iter = itertools.cycle(meshes_viz)
    this_geom = meshes[next(meshes_iter)]
    viz.add_geometry(this_geom, reset_bounding_box=False)

    print('#----------------------------------------------------------------')
    print('# launching mesh animation with the following properties')
    print('#--------')
    print('# number of meshes        :', len(meshes))
    print('# frames per anim cycle   :', num_frames)
    print('# twist axis              :', twist_axis)
    print('# twist step size    [deg]:', step_size)
    print('# twist angle        [deg]:', twist_angle)
    print('# field of view      [deg]:', viz.get_view_control().get_field_of_view())
    print('#----------------------------------------------------------------')

    # Run it!
    animation_counter = 0
    capture_counter = 0
    mov = None
    while viz.poll_events():
        if kpm.capture and not kpm.pause:
            capture_this_frame = False
            if capture_counter == 0:
                # starting capture
                # crop frames to have even dimensions (cannot resize visualizer)
                pcpi = viz.get_view_control().convert_to_pinhole_camera_parameters().intrinsic
                mov_height = pcpi.height if pcpi.height%2==0 else pcpi.height-1
                mov_width = pcpi.width if pcpi.width%2==0 else pcpi.width-1
                mov = np.zeros((num_frames, mov_height, mov_width, 3), dtype=np.uint8)
                print('# output mov dims         :', mov.shape)
                print('# output mov raw size [MB]:', np.prod(mov.shape)/1024/1024)
                print('# start capture')
                capture_this_frame = True
            elif 0 < capture_counter < num_frames:
                # continue capture
                capture_this_frame = True
            elif capture_counter == num_frames:
                # we are done
                print()
                print('# done capture')
                print('#--------')
                kpm.capture = False
                capture_counter = 0
            if capture_this_frame:
                print('# capture frame:', capture_counter, end="\r")
                rgb = viz.capture_screen_float_buffer(do_render=False)
                ix = animation_counter%num_frames
                mov[ix] = (np.array(rgb)*255).astype(np.uint8)[:mov_height, :mov_width]
                capture_counter += 1

        if not kpm.pause:
            # Update the mesh
            viz.remove_geometry(this_geom, reset_bounding_box=False)
            this_geom = meshes[next(meshes_iter)]
            viz.add_geometry(this_geom, reset_bounding_box=False)

            if twist_axis == 'y':
                viz.get_view_control().rotate(seq[animation_counter%len(seq)], 0, 0, 0)
            elif twist_axis == 'x':
                viz.get_view_control().rotate(0, seq[animation_counter%len(seq)], 0, 0)
            viz.update_renderer()
            animation_counter += 1
    viz.run()
    viz.destroy_window()
    return mov

def make_decimation_schedule(mx, mn, dec_steps, method='geometric'):
    """decimation schedule"""
    if method == 'geometric':
        out = (mx*(mn/mx)**(np.arange(dec_steps)/(dec_steps-1))).astype(int)
    elif method == 'linear':
        out = np.linspace(mx, mn, dec_steps).astype(int)
    return out

def incremental_decimation(mesh, dec_min, dec_steps, method='geometric'):
    """make a series of decimated meshes without redundant effort"""
    num_tri = np.array(mesh.triangles).shape[0]
    schedule = make_decimation_schedule(num_tri, dec_min, dec_steps, method)
    meshes = [mesh]
    for n in schedule[1:]:
        meshes.append(meshes[-1].simplify_quadric_decimation(n))
    return meshes

def make_trimesh(xyz):
    tri = scipy.spatial.Delaunay(xyz[:, :2])
    mesh = open3d.geometry.TriangleMesh()
    mesh.vertices = open3d.utility.Vector3dVector(xyz)
    mesh.triangles = open3d.utility.Vector3iVector(tri.simplices)
    mesh = add_bbox_points(mesh)
    return mesh

if __name__ == "__main__":
    pass


from typing import Callable, Optional
from addict import Dict
import torch
import torch.nn.functional as F

from gvcore.utils.types import TDataList
from gvcore.utils.misc import sharpen
from gvcore.model import GenericModule

from evaluator.segmentation import SegmentationEvaluator

class TSSegmentor(GenericModule):
    """
    Teacher-Student segmentor for semi-supervised learning.
    """

    def __init__(self, cfg, segmentor_class: Callable[..., GenericModule]):
        super(TSSegmentor, self).__init__(cfg)

        # Components ========
        self.segmentor = segmentor_class(cfg)
        self.ema_segmentor = segmentor_class(cfg)
        self.ema_segmentor.eval()

        # Initialize ========
        for param_q, param_k in zip(self.segmentor.parameters(), self.ema_segmentor.parameters()):
            param_k.data.copy_(param_q.data)
            param_k.requires_grad = False

        # Buffer ========
        for m in self.ema_segmentor.modules():
            if isinstance(m, (torch.nn.BatchNorm2d, torch.nn.SyncBatchNorm)):
                m.momentum = 0.9

        self.bn_statistics_pl = Dict()
        self.bn_statistics_eval = Dict()
        self.bn_statistics_mode = "eval"

        for name, buffer in self.ema_segmentor.named_buffers():
            self.bn_statistics_pl[name] = buffer.clone()
            self.bn_statistics_eval[name] = buffer.clone()

        # Tools ========
        self.tranining_evaluator = SegmentationEvaluator(
            num_classes=cfg.model.num_classes, distributed=cfg.num_gpus > 1, mode="window_avg", window_size=100
        )

    def to(self, device):
        super(TSSegmentor, self).to(device)
        for name, _ in self.ema_segmentor.named_buffers():
            self.bn_statistics_eval[name] = self.bn_statistics_eval[name].to(device)
            self.bn_statistics_pl[name] = self.bn_statistics_pl[name].to(device)

    def train(self, mode=True):
        self.training = mode
        self.segmentor.train(mode)
        self.ema_segmentor.eval()

    def switch_to_bn_statistics_pl(self):
        if self.bn_statistics_mode != "pl":
            for name, buffer in self.ema_segmentor.named_buffers():
                self.bn_statistics_eval[name] = buffer.clone()
                buffer.data = self.bn_statistics_pl[name]
            self.bn_statistics_mode = "pl"

    def switch_to_bn_statistics_eval(self):
        if self.bn_statistics_mode != "eval":
            for name, buffer in self.ema_segmentor.named_buffers():
                self.bn_statistics_pl[name] = buffer.clone()
                buffer.data = self.bn_statistics_eval[name]
            self.bn_statistics_mode = "eval"

    def forward_generic(self, *args, **kwargs):
        selector = kwargs.pop("selector")
        if selector == "student":
            return self.segmentor.forward_generic(*args, **kwargs)
        elif selector == "teacher":
            return self.ema_segmentor.forward_generic(*args, **kwargs)

    def forward_train(
        self,
        data_list_l_weak: TDataList,
        data_list_u_weak: TDataList,
        data_list_l_strong: TDataList,
        data_list_u_strong: TDataList,
    ):
        logits_l = self.forward_generic(data_list_l_strong, selector="student")
        # 1. Train with labeled data
        labels_l = torch.cat([data.label for data in data_list_l_weak], dim=0)
        loss_dict_l = self.get_losses(logits_l, labels_l, gt_labels=labels_l, labelled=True)

        # 2. Pseudo labeling and Consistency training with labeled and unlabeled data
        pseudo_labels_u = self._pseudo_labeling(data_list_u_weak)
        labels_u = torch.cat([data.label for data in data_list_u_weak], dim=0)
        logits_u = self.forward_generic(data_list_u_strong, selector="student")
        loss_dict_u = self.get_losses(logits_u, pseudo_labels_u, gt_labels=labels_u, labelled=False)

        loss_dict = Dict()
        loss_dict.loss_l = loss_dict_l
        loss_dict.loss_u = loss_dict_u

        # 3. Calculate mIoU of pseudo labels
        stat_dict = self._pseudo_labeling_statistics(pseudo_labels_u, labels_u)

        return loss_dict, stat_dict

    def get_losses(
        self,
        logits: torch.Tensor,
        labels: torch.Tensor,
        gt_labels: Optional[torch.Tensor] = None,
        labelled: bool = True,
    ):
        loss_dict = Dict()
        if labelled:
            loss = F.cross_entropy(logits, labels.long(), ignore_index=255)
        else:
            pseudo_probs_u, pseudo_preds_u = torch.max(labels, dim=1)
            mask = torch.ge(pseudo_probs_u, self.cfg.model.teacher.score_threshold)
            valid_mask = torch.ne(gt_labels, 255) & mask

            loss = F.cross_entropy(logits, pseudo_preds_u, reduction="none")
            loss = loss * valid_mask.float()

            loss = loss.sum() / (valid_mask.sum() + 1e-10)
        loss_dict.loss = loss

        return loss_dict

    @torch.no_grad()
    def _pseudo_labeling(self, data_list: TDataList, sharpen_factor: float = 0.5) -> torch.Tensor:
        # Update momentum batch statistic pl
        self.switch_to_bn_statistics_pl()
        self.ema_segmentor.train()
        self.forward_generic(data_list, selector="teacher")
        self.ema_segmentor.eval()

        logits = self.forward_generic(data_list, selector="teacher")
        probs = F.softmax(logits, dim=1)
        if sharpen_factor > 0:
            probs = sharpen(probs, sharpen_factor)

        self.switch_to_bn_statistics_eval()
        return probs.detach()

    @torch.no_grad()
    def _pseudo_labeling_statistics(self, pseudo_labels: torch.Tensor, gt_labels: torch.Tensor) -> Dict:
        if pseudo_labels.dim() == 4:
            pseudo_labels = torch.argmax(pseudo_labels, dim=1)

        self.tranining_evaluator.process(pseudo_labels, gt_labels)
        metrics = self.tranining_evaluator.calculate()

        return Dict(PL_mIoU=metrics["mIoU"])

    @torch.no_grad()
    def _momentum_update(self, m: float):
        """
        Momentum update of the key encoder
        """
        for buffer_q, buffer_k in zip(self.segmentor.buffers(), self.ema_segmentor.buffers()):
            buffer_k.data = buffer_k.data * m + buffer_q.data * (1.0 - m)

        for param_q, param_k in zip(self.segmentor.parameters(), self.ema_segmentor.parameters()):
            param_k.data = param_k.data * m + param_q.data * (1.0 - m)

    @torch.no_grad()
    def forward_eval(self, data_list: TDataList):
        # 1. Test student
        logits_stu = self.forward_generic(data_list, selector="student")
        probs_stu = F.softmax(logits_stu, dim=1)
        preds_stu = torch.argmax(probs_stu, dim=1)

        # 2. Test teacher
        logits_tea = self.forward_generic(data_list, selector="teacher")
        probs_tea = F.softmax(logits_tea, dim=1)
        preds_tea = torch.argmax(probs_tea, dim=1)

        return preds_stu, preds_tea


import pytest
from pyspark.sql import DataFrame as SparkDataFrame
from pyspark.sql import SparkSession
from pyspark.sql import functions as sf
from pyspark_test import assert_pyspark_df_equal

from feature_catalog.base_feature_group import BaseFeatureGroup
from feature_catalog.utils import MissingColumnError, UnsupportedAggregationLevel

class GroupA(BaseFeatureGroup):

    supported_levels = {"id"}
    available_features = {"feature_3", "feature_4"}

    def _compute_feature_group(
        self, spark: SparkSession, intermediate_features: SparkDataFrame, aggregation_level: str
    ) -> SparkDataFrame:
        return (
            spark.createDataFrame([(1,), (2,), (3,), (4,)], schema="id integer")
            .withColumn("feature_3", sf.lit("f3"))
            .withColumn("feature_4", sf.lit("f4"))
        )

@pytest.fixture
def features(spark):
    return spark.createDataFrame(
        [
            dict(id=1, feature_1="a", feature_2="dummy"),
            dict(id=2, feature_1="a", feature_2="dummy"),
            dict(id=3, feature_1="b", feature_2="dummy"),
            dict(id=4, feature_1="c", feature_2=""),
        ]
    )

@pytest.fixture
def features_extended_expected(features):
    return features.withColumn("feature_3", sf.lit("f3")).withColumn("feature_4", sf.lit("f4"))

def test_extend(spark: SparkSession, features: SparkDataFrame, features_extended_expected: SparkDataFrame):
    features_extended = GroupA(features_of_interest=GroupA.available_features, aggregation_level="id").extend(
        spark=spark, features=features
    )
    assert_pyspark_df_equal(features_extended, features_extended_expected)

def test_extend__missing_column(spark: SparkSession):
    group_a = GroupA(features_of_interest=GroupA.available_features, aggregation_level="id")
    group_a.extend(spark=spark, features=spark.createDataFrame([], schema="id integer"))

    with pytest.raises(MissingColumnError):
        group_a.extend(spark=spark, features=spark.createDataFrame([], schema="not_id integer"))

def test__init__():
    GroupA(features_of_interest=GroupA.available_features, aggregation_level="id")

    with pytest.raises(UnsupportedAggregationLevel):
        GroupA(features_of_interest=GroupA.available_features, aggregation_level="unsupported_level")