#!/usr/bin/env python
# -*- coding:utf-8 -*-

# Description:
# Date: 2021/12/8 10:59
# Author: dupei

import argparse
import os
import pickle
import platform
import shutil
import torch
import DataSet
from evaluations import pairwise_similarity
from util import extract_feature, load_image, create_model
from utils.serialization import load_checkpoint

def save_retrieved_images(gallery, topk_index, topk_sim, output_dir):
    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)
    os.makedirs(output_dir, exist_ok=True)
    for i in range(len(topk_index)):
        image_path = gallery[topk_index[i]][2]
        _, filename = os.path.split(image_path)
        name, ext = os.path.splitext(filename)
        shutil.copyfile(image_path, os.path.join(output_dir, '%d_' % i + name + '_%f' % topk_sim[i] + ext))

def parse_args():
    parser = argparse.ArgumentParser(description='PyTorch Testing')

    parser.add_argument('--data', type=str, default='cub')
    parser.add_argument('--data_root', type=str, default='data/Oxford')

    parser.add_argument('--net', type=str, default='BN-Inception')
    parser.add_argument('--resume', '-r', type=str, default='ckp_ep600.pth.tar', metavar='PATH')

    parser.add_argument('--dim', '-d', type=int, default=512,
                        help='Dimension of Embedding Feather')
    parser.add_argument('--width', type=int, default=227,
                        help='width of input image')

    parser.add_argument('--batch_size', type=int, default=100)
    parser.add_argument('--nThreads', '-j', default=8, type=int, metavar='N',
                        help='number of data loading threads (default: 2)')

    parser.add_argument('--topk', '-k', type=int, default=10, help='top-k')
    parser.add_argument('--query_img', '-query', type=str, help='path to query image')
    parser.add_argument('--gallery_pickle', '-gallery', type=str, help='path to gallery image feature pickle file',
                        default='oxford_gallery_feature.pickle')
    parser.add_argument('--output', type=str, default='tmp/retrieved', help='path to output dir')

    args = parser.parse_args()
    return args

def main(args):
    if platform.system() == 'Windows':
        torch.multiprocessing.freeze_support()

    checkpoint = load_checkpoint(args.resume)

    # 修改state_dict中的keys，去除module.
    from collections import OrderedDict
    new_state_dict = OrderedDict()
    for name, param in checkpoint['state_dict'].items():
        new_state_dict[name.replace('module.', '')] = param
    checkpoint['state_dict'] = new_state_dict

    model = create_model(net=args.net, dim=args.dim, checkpoint=checkpoint)

    # data = DataSet.create(args.data, width=args.width, root=args.data_root)
    data = DataSet.create_general_dataset(root=args.data_root)

    # gallery_loader = torch.utils.data.DataLoader(
    #     data.gallery, batch_size=args.batch_size, shuffle=False,
    #     drop_last=False, pin_memory=True, num_workers=args.nThreads)
    # gallery_feature, gallery_labels = extract_features(model, gallery_loader, print_freq=1e5, metric=None)

    with open(args.gallery_pickle, 'rb') as handle:
        gallery_feature = pickle.load(handle)

    # query_loader = torch.utils.data.DataLoader(
    #     data.query, batch_size=args.batch_size,
    #     shuffle=False, drop_last=False,
    #     pin_memory=True, num_workers=args.nThreads)
    # query_feature, query_labels = extract_features(model, query_loader, print_freq=1e5, metric=None)

    images = load_image(args.query_img)
    query_feature = extract_feature(model, images)

    sim_mat = pairwise_similarity(query_feature, gallery_feature)
    topk_result = torch.topk(sim_mat, args.topk, dim=1, largest=True, sorted=True)
    topk_sim = topk_result[0]
    topk_index = topk_result[1]

    topk_sim = torch.squeeze(topk_sim).detach().numpy()
    topk_index = torch.squeeze(topk_index).detach().numpy()

    os.makedirs(args.output, exist_ok=True)
    save_retrieved_images(data.gallery, topk_index, topk_sim, args.output)

# --data_root data\CUB_200_2011 --query_img data\CUB_200_2011\train\010.Red_winged_Blackbird\Red_Winged_Blackbird_0001_3695.jpg --gallery_pickle cub_train_feature.pickle --output tmp\cub_retrieved -k 20
# --data_root data/Oxford --query_img data/Oxford/query/all_souls_000013.jpg --output tmp/retrieved -k 20
if __name__ == '__main__':
    main(parse_args())


def get_fee():
    '''Allows you to update fee. Returns the fee.'''

    global fee  # This will be the trasaction fee - or I guess more specifically,
                # it's what we multiply each starting transaction amount by to account for the fee.
    try:
        fee = float(input('What is the transaction fee? (as a percentage): '))
    except:
        print("Input invalid. Please try again, or type '111' to use the default amount.")
        get_fee()
        return

    if fee == 111:
        fee = 0.9974
    else:
        fee = 1 - fee / 100
        return fee

fee = 0.9974


import os
import json
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from log_module import ProjectLogger
from flight_data import FlightData

class DataManager:
    """
        This class is responsible for talking to the Google Spreadsheet.
    """
    API_SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
    API_SERVICE_NAME = "sheets"
    API_VERSION = "v4"
    AVAILABLE_RANGE = 'A:B'

    def __init__(self):
        # create new logger for DataManager
        self._logger = ProjectLogger().get_module_logger('DataManager')
        self._config = self._check_if_config_folder_exists()
        self._logger.debug('Initializing DataManager...')
        self._spreadsheet_id = self._config['google_sheet_id']
        self._handle_credentials()
        try:
            # Build the Google service
            service = build(DataManager.API_SERVICE_NAME, DataManager.API_VERSION, credentials=self._creds)
            # Call the Sheets API
            self._sheet = service.spreadsheets()
            self._logger.debug('Google Sheet service created')
        except HttpError as err:
            self._logger.error(err)

        self._logger.debug('DataManager initialized')

    @staticmethod
    def _prepare_range(sheet_id: str, cells: str) -> str:
        """
        This method prepares the range for the Google Sheet API
        :param sheet_id: Sheet id to be updated e.g. 'Sheet1'
        :param cells: Cells to be updated e.g. 'A1:B2'
        :return: Range e.g. 'Sheet1!A1:B2'
        """
        return f'\'{sheet_id}\'!{cells}'

    @staticmethod
    def _prepare_sheet_id(start_location: str) -> str:
        """
        This method prepares the sheet id for the Google Sheet API
        :param start_location: start location IATA code
        :return: Ready to use sheet id for application e.g. 'FROM_GDA'
        """
        return f'FROM_{start_location}'

    @staticmethod
    def get_flight_dict(start_location: str, destination: str, price: int) -> dict:
        """
        This method creates the flight data dictionary
        :param start_location: start location IATA code
        :param destination: destination IATA code
        :param price: price of the flight
        :return: proper data dictionary
        """
        return {'origin_airport': start_location, 'destination_airport': destination, 'price': price}

    def _handle_credentials(self) -> None:
        """
        This method handles the Google credentials for the application.
        It checks if the token file exists, if it does, it loads the credentials from file.
        If there are no credentials, it creates new ones.
        :return: None
        """
        # Check if token file exists, if it does, load credentials from file
        if os.path.exists(self._config['google_token_file']):
            self._creds = Credentials.from_authorized_user_file(self._config['google_token_file'],
                                                                DataManager.API_SCOPES)
            self._logger.debug('Google credentials loaded from file')

        # If there are no (valid) credentials available, let the user log in using OAuth.
        if not self._creds or not self._creds.valid:
            if self._creds and self._creds.expired and self._creds.refresh_token:
                # If credentials are expired, but refresh token is available, refresh credentials
                self._logger.debug('Credentials expired, refreshing...')
                self._creds.refresh(Request())
                self._logger.debug('Credentials refreshed')
            else:
                # If credentials are not available, create new credentials
                self._logger.debug('Credentials not available, creating new...')
                flow = InstalledAppFlow.from_client_secrets_file(self._config['google_credentials_file'],
                                                                 DataManager.API_SCOPES)
                self._creds = flow.run_local_server(port=0)
                self._logger.debug('Credentials created')
            # Save the credentials for the next run
            with open(self._config['google_token_file'], 'w+') as token:
                token.write(self._creds.to_json())
                self._logger.debug('Credentials saved to file')

    def _check_if_config_folder_exists(self) -> dict:
        """
        This function checks if config folder exists. If not, it creates it.
        :return: config dictionary
        """
        config = {
            'tequila_api_key': '',
            'google_token_file': '.config/token.json',
            'google_credentials_file': '.config/client_secret.json',
            'google_sheet_id': ''
        }

        if not os.path.exists('.config/'):
            self._logger.warning('Config folder does not exist, creating...')
            os.mkdir('.config')

        if not os.path.exists('.config/config.json'):
            self._logger.warning('Config file does not exist, creating...')

            with open('.config/config.json', 'w+') as f:
                f.write(json.dumps(config))

        else:
            with open('.config/config.json', 'r') as f:
                config = json.load(f)

        if not os.path.exists(config['google_token_file']):
            self._logger.warning('Google token file does not exist, creating...')

            with open(config['google_token_file'], 'w+') as f:
                f.write('')

        if not os.path.exists(config['google_credentials_file']):
            self._logger.error(
                'Google credentials file does not exist! Please download it from Google Cloud Platform and put it in .config folder. Name it `client_secret.json` and try again.')

            raise FileNotFoundError('Google credentials file not found! More info in log file.')

        return config

    def get_config(self) -> dict:
        """
        This method returns the json setup for the application
        :return: config dictionary with `tequila_api_key`, `google_token_file`, `google_credentials_file` and `google_sheet_id`
        """
        return self._config

    def _create_sheet(self, sheet_id: str) -> bool:
        """
        This method creates the sheet in the connected Google spreadsheet
        :param sheet_id: Sheet id to be created e.g. 'Sheet1'
        :return: True if sheet was created, False otherwise
        """
        status = False

        self._logger.debug(f'Creating sheet {sheet_id}')

        try:
            # Call the Sheets API to create the sheet
            self._logger.debug(f'Calling Sheets API to create sheet {sheet_id}')
            self._sheet.batchUpdate(spreadsheetId=self._spreadsheet_id, body={
                'requests': [
                    {
                        'addSheet': {
                            'properties': {
                                'title': sheet_id
                            }
                        }
                    }
                ]
            }).execute()
            status = True
        except HttpError as err:
            self._logger.error(err)

        return status

    def _check_if_sheet_exists(self, sheet_id: str) -> bool:
        """
        This method checks if the sheet exists in the connected Google spreadsheet
        :param sheet_id: Sheet id to be checked e.g. 'Sheet1'
        :return: True if a sheet exists, False otherwise
        """
        status = False

        self._logger.debug(f'Checking if sheet {sheet_id} exists')

        # Call the Sheets API to get the list of sheets
        sheets = self.get_start_locations()

        # Check if sheet exists
        for sheet in sheets:
            if f'FROM_{sheet}' == sheet_id:
                status = True
                break

        return status

    def _append_data(self, sheet_id: str, new_values: list) -> bool:
        """
        This method appends the data to the connected Google spreadsheet
        :param sheet_id: Sheet id to be updated e.g. 'Sheet1'
        :param new_values: 2D list of values to append e.g. [['New Value A', 'New Value B'], ['New Value C', 'New Value D']]
        :return: True if append was successful, False otherwise
        """
        status = False

        self._logger.debug(f'Appending values {new_values} to sheet {sheet_id}')

        # Add sheet id to the range, so it looks like 'Sheet1!A1:B2'
        data_range = self._prepare_range(sheet_id, DataManager.AVAILABLE_RANGE)

        self._logger.debug(f'Calling Sheets API to append data to {data_range}')

        try:
            # Call the Sheets API to append the data
            self._sheet.values().append(spreadsheetId=self._spreadsheet_id, range=data_range,
                                        valueInputOption='USER_ENTERED', insertDataOption='INSERT_ROWS',
                                        body={'values': new_values}).execute()
            status = True
        except HttpError as err:
            self._logger.error(err)

        return status

    def _update_data(self, new_values: list, cells_range: str) -> bool:
        """
        This method updates the data in the connected Google spreadsheet
        :param new_values: 2D list of values to update [['New Value A', 'New Value B'], ['New Value C', 'New Value D']]
        :param cells_range: Range to update in Sheet!A1:B2 format --> 'My Custom Sheet!A1:B2'
        :return: True if update was successful, False otherwise
        """
        status = False
        self._logger.debug(f'Updating range `{cells_range}` with values {new_values}')
        try:
            # Call the Sheets API to update the data
            result = self._sheet.values().update(spreadsheetId=self._spreadsheet_id, range=cells_range,
                                                 valueInputOption='RAW',
                                                 body={'values': new_values}).execute()
            # Check if any cells were updated
            status = result.get('updatedCells') > 0
            if status:
                self._logger.debug(f'{result.get("updatedCells")} cells updated')
            else:
                self._logger.debug(f'No cells updated')
        except HttpError as err:
            self._logger.error(err)

        return status

    def _read_data(self, cells_range: str = None) -> list:
        """
        This method reads the data from the connected Google spreadsheet
        :param cells_range: Range to read in Sheet!A1:B2 format e.g. 'My Custom Sheet!A1:B2'
        :return: 2D list of values [['Value A', 'Value B'], ['Value C', 'Value D']]
        """
        # Check if range is specified
        if cells_range is None:
            self._logger.error('No range specified, using default')
            cells_range = DataManager.AVAILABLE_RANGE

        try:
            # Call the Sheets API to read the data
            self._logger.debug('Calling Sheets API to access data')

            # Check if sheet exists
            sheet_id = cells_range.split("!")[0].replace('\'', '')
            if not self._check_if_sheet_exists(sheet_id):
                self._logger.error(f'Sheet `{sheet_id}` does not exist')
                return []

            result = self._sheet.values().get(spreadsheetId=self._spreadsheet_id, range=cells_range).execute()
            values = result.get('values', [])
        except HttpError as err:
            self._logger.error(err)
            values = []
        return values

    def _get_flights_amount(self, start_location: str) -> int:
        """
        This method reads the amount of flights from the connected Google spreadsheet
        :param start_location: start location IATA code
        :return: amount of flights
        """
        self._logger.debug(f'Reading flights amount for {start_location}...')
        # Call the Sheets API to read the data
        sheet_id = self._prepare_sheet_id(start_location)
        read_range = self._prepare_range(sheet_id, DataManager.AVAILABLE_RANGE)
        flights = self._read_flights(read_range)

        # Return amount of flights
        self._logger.debug(f'Amount of flights for {start_location}: {len(flights)}')

        return len(flights)

    def _read_flights(self, start_location: str) -> dict:
        """
        This method reads all flights data from the connected Google spreadsheet
        :param start_location: start location IATA code
        :return: dictionary of flights data {'iataCode1': 'lowestPrice1', 'iataCode2': 'lowestPrice2'}
        """
        self._logger.debug(f'Reading flights data...')
        # Call the Sheets API to read the data
        sheet_id = self._prepare_sheet_id(start_location)
        read_range = self._prepare_range(sheet_id, DataManager.AVAILABLE_RANGE)
        values = self._read_data(read_range)

        self._logger.debug(f'Collected all flights data')

        self._logger.debug(values)

        # Return dictionary of flight data {'iataCode1': 'lowestPrice1', 'iataCode2': 'lowestPrice2'}
        flights = {f'{row[0]}': int(row[1]) for row in values if len(row) == 2}

        return flights

    def _add_flight(self, flight: FlightData) -> None:
        """
        This method adds the new flight data to the connected Google spreadsheet
        :param flight: Flight data to add e.g. {'start_iata_code': 'ABC', dest_iata_code': 'ABC', 'lowestPrice': 123}
        :return: True if data added, False otherwise
        """
        self._logger.debug(
            f'Adding flight data: {flight.origin_airport}->{flight.destination_airport} with price {flight.price}$'
        )

        # Check if there is data to add
        if flight is None:
            self._logger.error('add_flight: No data to add')
            return

        # Check if sheet exists
        sheet_id = self._prepare_sheet_id(flight.origin_airport)
        if not self._check_if_sheet_exists(sheet_id):
            self._create_sheet(sheet_id)

        # Call the Sheets API to add the data
        status = self._append_data(sheet_id, [[flight.destination_airport, flight.price]])

        if status:
            self._logger.debug(f'Flight data added')
        else:
            self._logger.error(f'Flight data not added')

    def get_start_locations(self) -> list:
        """
        This method reads all start_locations from the connected Google spreadsheet
        :return: list of start locations IATA codes
        """
        # Call the Sheets API to get the list of sheets
        try:
            self._logger.debug(f'Calling Sheets API to get the list of sheets')
            result = self._sheet.get(spreadsheetId=self._spreadsheet_id).execute()
            sheets = result.get('sheets', [])
            self._logger.debug(f'{len(sheets)} sheets found')
        except HttpError as err:
            self._logger.error(err)
            sheets = []

        return [sheet['properties']['title'].replace('FROM_', '') for sheet in sheets]

    def get_destination_airports(self, start_location: str) -> list:
        """
        This method reads all destinations for given start location from the connected Google spreadsheet
        :param start_location: start location IATA code
        :return: list of destinations IATA codes
        """
        self._logger.debug(f'Reading destinations for {start_location}...')
        # Call the Sheets API to read the data
        sheet_id = self._prepare_sheet_id(start_location)
        read_range = self._prepare_range(sheet_id, DataManager.AVAILABLE_RANGE)
        values = self._read_data(read_range)
        destinations = [row[0] for row in values]

        # Return list of destinations
        self._logger.debug(f'Destinations for {start_location}: {destinations}')
        return destinations

    def get_flight_data(self, data: FlightData) -> dict:
        """
        This method reads the flight data (by iata code) from the connected Google sheet
        :param data: See FlightData class for more information.
        :return: dictionary of certain flight data {'iataCode': 'lowestPrice'}
        :raises ValueError: if there is no flight data for given start and destination
        """
        flights = self._read_flights(start_location=data.origin_airport)

        if data.destination_airport not in flights.keys():
            self._logger.error(f'No flight data found for {data.origin_airport}->{data.destination_airport}')
            return {}

        return flights.get(data.destination_airport)

    def update_flight(self, flight: FlightData) -> int:
        """
        This method updates the flight data in the connected Google sheet
        :param flight: See FlightData class for more information.
        :return: status code (0 - updated with lower price, 1 - not updated)
        :raises ValueError: if error occurred during update process
        """
        status = False

        self._logger.info(
            f'Updating flight data: {flight.origin_airport}->{flight.destination_airport} with price {flight.price}'
        )

        # Check if start location exists in the sheet
        if not self._check_if_sheet_exists(self._prepare_sheet_id(flight.origin_airport)):
            self._logger.debug(
                f'update_flight: There is no sheet for iata code `{flight.origin_airport}`, creating new')
            self._add_flight(flight)
            return 0

        # Get range to update
        row_number = -1
        existing_values = self._read_flights(flight.origin_airport)
        for nr, row in enumerate(existing_values.keys()):
            if row == flight.destination_airport:
                self._logger.debug(f'Found existing entry for `{flight.destination_airport}`')
                row_number = nr + 1
                break

        # Add new data if there is no data for this destination
        if row_number == -1:
            self._logger.debug(f'update_flight: No data for iata code `{flight.destination_airport}`')
            self._add_flight(flight)
            return 0

        # Call the Sheets API to update the data
        sheet_id = self._prepare_sheet_id(flight.origin_airport)
        update_range = self._prepare_range(sheet_id, f'B{row_number}')

        # Check if price is lower than existing one
        if flight.price < existing_values[flight.destination_airport]:
            self._logger.debug(f'update_flight: New price is lower than existing one, updating...')
            status = self._update_data([[flight.price]], update_range)
        else:
            self._logger.debug(f'update_flight: New price is higher than existing one, not updating...')

        return 0 if status else 1


from http import HTTPStatus

import allure
import pytest
from allure_commons.types import LinkType
from jsonschema import validate

from base.api.courses import get_courses, get_course, delete_course, update_course, create_course, \
    courses_token_required, courses_send_file
from models.courses import CreateCourse, UpdateCourse, Course

# http://localhost:50853/index.html#behaviors/362548e62425745a6269d682cefebd3e/705479de203ca840/

@pytest.mark.allure_homework
@allure.severity(allure.severity_level.NORMAL)
@allure.epic('API')
@allure.feature('Courses')
@pytest.mark.courses
class TestCourses:
    schema = Course.manager.to_schema

    @allure.title('Getting all courses')
    @allure.description('В этом тесте мы получаем все курсы')
    @allure.story('Getting courses')
    @allure.issue(url='https://www.atlassian.com/software/jira', name='URGY-1025')
    @allure.link(
        url='https://www.artlebedev.ru/orfograf/',
        link_type=LinkType.TEST_CASE,
        name='Some test case'
    )
    def test_get_all_courses(self):
        get_courses_response = get_courses()
        get_courses_response_json = get_courses_response.json()
        schema = Course.manager.to_array_schema

        assert get_courses_response.status_code == HTTPStatus.OK
        validate(instance=get_courses_response_json, schema=schema)

    @allure.title('Creating course')
    @allure.description('В этом тесте мы создаем курс')
    def test_create_course_view(self):
        body = CreateCourse.manager.to_dict()

        create_course_response = create_course(body)
        create_course_response_json = create_course_response.json()

        with allure.step('Проверяем статус-код'):
            assert create_course_response.status_code == HTTPStatus.OK
        with allure.step('Проверяем title из response'):
            assert create_course_response_json['title'] == body['title']
        with allure.step('Проверяем description из response'):
            assert create_course_response_json['description'] == body['description']
        with allure.step('Проверяем image из response'):
            assert create_course_response_json['image'] == body['image']

        validate(instance=create_course_response_json, schema=self.schema)

    @allure.story('Getting courses')
    @allure.title('Getting course')
    @allure.description('В этом тесте мы получаем курс')
    def test_get_course(self, course_function):
        get_course_response = get_course(course_function['id'])
        get_course_response_json = get_course_response.json()

        with allure.step('Проверяем статус-код'):
            assert get_course_response.status_code == HTTPStatus.OK
        with allure.step('Валидируем response'):
            validate(instance=get_course_response_json, schema=self.schema)

    @allure.title('Updating course')
    @allure.description('В этом тесте мы обновляем курс')
    def test_update_course(self, course_function):
        body = UpdateCourse.manager.to_dict()

        update_course_response = update_course(course_function['id'], body)
        update_course_response_json = update_course_response.json()

        assert update_course_response.status_code == HTTPStatus.OK
        assert update_course_response_json['title'] == body['title']

        validate(instance=update_course_response_json, schema=self.schema)
        allure.attach.file('./files/hello.json', name='Some json response', attachment_type=allure.attachment_type.JSON)
        allure.attach.file('./files/hello.txt', name='Json  txt response', attachment_type=allure.attachment_type.TEXT)

    @allure.title('Deleting course')
    @allure.description('В этот тесте мы удаляем курс и проверяем, что он удален')
    def test_delete_course(self, course_function):
        deleted_course_response = delete_course(course_function['id'])

        assert deleted_course_response.status_code == HTTPStatus.NO_CONTENT
        allure.attach.file('./files/used.jpg', name='Some photo', attachment_type=allure.attachment_type.JPG)

    @allure.description('В этот тесте мы проверяем, что при вводе несуществующей почты, мы не получим токен,'
                        'с помощью которого можно получить доступ к курсу')
    @pytest.mark.parametrize('email', ['blablabla', 'abracadabra', 'daria.pishchulinana'])
    def test_courses_token_required(self, email):
        body = {
            "email": f"{email}@gmail.com"
        }
        courses_token_response = courses_token_required(body)
        courses_token_response_json = courses_token_response.json()
        assert courses_token_response.status_code == HTTPStatus.OK

        if email == 'daria.pishchulinana':
            allure.dynamic.title('Positive test')
            assert courses_token_response_json['success'] == 'yes'
        else:
            allure.dynamic.title('Negative test')
            assert courses_token_response_json['success'] == 'no'

    @allure.title('Sending file')
    @allure.description('В этот тесте мы отправляем файлы разных форматов')
    @allure.severity(allure.severity_level.MINOR)
    def test_courses_send_file(self):
        send_file_response = courses_send_file()
        send_file_response_json = send_file_response.json()

        with allure.step('Проверяем статус-код'):
            assert send_file_response.status_code == HTTPStatus.OK
        with allure.step('Проверяем название файла'):
            assert send_file_response_json['filename'] == 'hello.txt'
        with allure.step('Проверяем тип данных названия файла'):
            assert isinstance(send_file_response_json['filename'], str) # лучше порверять через json schema


from typing import Dict, List, Optional, Iterable, Tuple

import abc
from . import dates
from . import paths
from .paths import Basis
from .step import Step

class ConfigBase(metaclass=abc.ABCMeta):

    distribution = "gev"

    def set_fit_years(self, start_year: Optional[int], end_year: Optional[int]):
        assert (start_year is None) == (end_year is None)
        if start_year:
            self.result_fit_years = lambda: range(start_year, end_year + 1)

    def set_integration_windows(self, windows):
        if windows:
            self.integration_windows = lambda: windows

    def set_distribution(self, distribution: Optional[str] = None):
        if distribution:
            self.distribution = distribution

    def land_mask(self) -> Optional[paths.Vardef]:
        """
        An optional land mask for composite outputs
        """
        return None

    @abc.abstractmethod
    def historical_years(self):
        """
        Provides a list of the years of historical record available for use
        during spin-up.
        """
        return []

    def historical_yearmons(self):
        """
        Provides all YYYYMM time steps within the historical period
        """
        return [dates.format_yearmon(year, month)
                for year in self.historical_years()
                for month in dates.all_months]

    @abc.abstractmethod
    def result_fit_years(self) -> Iterable[int]:
        """
        Provides a list of years of data to be considered in fitting distributions
        for computed variables.
        """
        return []

    def result_fit_yearmons(self) -> List[str]:
        """
        Provides all YYYYMM time steps within the result fitting period
        """
        return [dates.format_yearmon(year, month)
                for year in self.result_fit_years()
                for month in dates.all_months]

    @abc.abstractmethod
    def static_data(self):
        pass

    @abc.abstractmethod
    def workspace(self) -> paths.DefaultWorkspace:
        pass

    @abc.abstractmethod
    def observed_data(self) -> paths.ObservedForcing:
        """
        Returns a Forcing instance capable of providing data for a given YYYYMM
        :return:
        """
        pass

    def forecast_data(self, model: str) -> paths.ForecastForcing:
        """
        Returns a Forcing instance capable of providing data for a given YYYYMM/forecast target/ensemble member
        :return:
        """
        pass

    def global_prep(self) -> List[Step]:
        """
        Returns a (possibly empty) list of steps that are included exactly once in the Makefile, regardless
        of which time steps/forecasts/etc. are also present in the Makefile.
        :return:
        """
        steps = self.static_data().global_prep_steps() + self.observed_data().global_prep_steps()
        for model in self.models():
            steps += self.forecast_data(model).global_prep_steps()
        return steps

    def should_run_spinup(self) -> bool:
        """
        Indicates whether this configuration requires a spinup phase.
        :return:
        """
        return True

    def should_run_lsm(self, yearmon: Optional[str]=None) -> bool:
        return True

    def result_postprocess_steps(self, model: str = None, yearmon: str = None, target: str = None, member: str = None) -> List[Step]:
        """
        Provides a list of one or more postprocessing steps to be applied to LSM results
        for a given YYYYMM/forecast target/ensemble member
        """
        return []

    def forecast_targets(self, yearmon: str) -> List[str]:
        """
        Provides a list of forecast target YYYYMM values for a given YYYYMM, or an empty
        list if the configuration does not contain forecasts.
        """
        return []

    def models(self) -> List[str]:
        """
        Provides a list of forecast models used in this configuration.
        """
        return []

    def forecast_ensemble_members(self, model: str, yearmon: str, *, lag_hours: Optional[int] = None) -> List[str]:
        """
        Provides a list of forecast ensemble members for a given YYYYMM, or
        an empty list if the configuration does not contain forecasts.
        If `lag_hours` is provided, only return ensemble members generated
        within more than `lag_hours` from present time.
        """
        return []

    def weighted_members(self, yearmon: str) -> Iterable[Tuple[str, str, float]]:
        for model in self.models():
            members = self.forecast_ensemble_members(model, yearmon)
            for member in members:
                weight = 1.0/len(members)/len(self.models())
                yield model, member, weight

    @staticmethod
    def integration_windows() -> List[int]:
        """
        Provides a list of integration windows (in months)
        """
        return [3, 6, 12, 24, 36, 60]

    @staticmethod
    def forcing_rp_vars(*, basis: Optional[Basis]=None) -> List[str]:
        """
        Provides a list of forcing variables for which return periods should be calculated
        """
        if not basis:
            return [
                'T',
                'Pr'
            ]

        if basis == Basis.BASIN:
            return [
                'T',
                'Pr'
            ]

        assert False

    @staticmethod
    def lsm_rp_vars(*, basis: Optional[Basis]=None) -> List[str]:
        """
        Provides a list of LSM output variables for which return periods should be calculated
        """

        if not basis:
            return [
                'Bt_RO',
                'PETmE',
                'PET',
                'P_net',
                'RO_mm',
                'Sa',
                'Sm',
                'Ws'
            ]

        if basis == Basis.BASIN:
            return [
                'Bt_RO'
            ]

        assert False

    @staticmethod
    def state_rp_vars(*, basis: Optional[Basis]=None) -> List[str]:
        """
        Provides a list of state variables for which return periods should be calculated
        """

        if not basis:
            return['Snowpack']

        if basis == Basis.BASIN:
            return []

    @classmethod
    def forcing_integrated_vars(cls, basis: Optional[Basis]=None) -> Dict[str, List[str]]:
        """
        Provides a dictionary whose keys are forcing variables to be time-integrated, and whose
        values are lists of stats to apply to each of those variables (min, max, ave, etc.)
        """
        if not basis or basis == Basis.BASIN:
            return {
                'Pr'   : ['sum'],
                'T'    : ['ave']
                }

        assert False

    @classmethod
    def lsm_integrated_vars(cls, basis: Optional[Basis]=None) -> Dict[str, List[str]]:
        """
        Provides a dictionary whose keys are LSM output variables to be time-integrated, and whose
        values are lists of stats to apply to each of those variables (min, max, ave, etc.)
        """

        if not basis:
            return {
                'Bt_RO': ['min', 'max', 'sum'],
                'E': ['sum'],
                'PETmE': ['sum'],
                'P_net': ['sum'],
                'RO_mm': ['sum'],
                'Sa'   : ['sum'],
                'Ws': ['ave'],
            }

        if basis == Basis.BASIN:
            return {
                'Bt_RO': ['sum']
            }

        assert False

    @classmethod
    def lsm_integrated_stats(cls, basis: Optional[Basis]=None) -> Dict[str, List[str]]:
        """
        Provides a dictionary whose keys are stat names and whose values are a list of variables
        two which that stat should be applied. It can be thought of as the inverse of lsm_integrated_vars()
        """
        integrated_stats = {}

        for var, varstats in cls.lsm_integrated_vars(basis=basis).items():
            for stat in varstats:
                if stat not in integrated_stats:
                    integrated_stats[stat] = []
                integrated_stats[stat].append(var)

        return integrated_stats

    @classmethod
    def lsm_integrated_var_names(cls, basis: Optional[Basis]=None) -> List[str]:
        """
        Provides a flat list of time-integrated variable names
        """
        return [var + '_' + stat for var, stats in cls.lsm_integrated_vars(basis=basis).items() for stat in stats]

    @classmethod
    def forcing_integrated_stats(cls, basis: Optional[Basis]=None) -> Dict[str, List[str]]:
        """
        Provides a dictionary whose keys are stat names and whose values are a list of variables
        two which that stat should be applied. It can be thought of as the inverse of forcing_integrated_vars()
        """
        integrated_stats = {}

        for var, varstats in cls.forcing_integrated_vars(basis=basis).items():
            for stat in varstats:
                if stat not in integrated_stats:
                    integrated_stats[stat] = []
                integrated_stats[stat].append(var)

        return integrated_stats

    @classmethod
    def forcing_integrated_var_names(cls, basis: Optional[Basis]=None) -> List[str]:
        """
        Provides a flat list of time-integrated variable names
        """
        return [var + '_' + stat for var, stats in cls.forcing_integrated_vars(basis=basis).items() for stat in stats]

    @classmethod
    def all_integrated_stats(cls, basis: Optional[Basis]=None) -> Dict[str, List[str]]:
        """
        Provides a dictionary whose keys are stat names and whose values are a list of variables
        two which that stat should be applied. This combines the inverse of forcing_integrated_vars()
        and lsm_integrated_vars().
        """
        integrated_stats = cls.lsm_integrated_stats(basis=basis)

        for var, varstats in cls.forcing_integrated_vars(basis=basis).items():
            for stat in varstats:
                if stat not in integrated_stats:
                    integrated_stats[stat] = []
                integrated_stats[stat].append(var)

        return integrated_stats


import pandas as pd
import sklearn.cluster, sklearn.preprocessing
from sklearn import metrics

alco2009 = pd.read_csv("niaaa-report2009.csv", index_col="State")
columns = ["Wine", "Beer"]
# 클러스터링 객체를 생성하고 모델을 학습시킨다.
kmeans = sklearn.cluster.KMeans(n_clusters=9)
kmeans.fit(alco2009[columns])
# cluster 변수는 사람의 직관에 의해 설정되고 수행된 결과에 따라 재조정한다.
# 클러스터링된 결과가 의미가 있다고 판단했을 때 최종 클러스터링 변수 확정
alco2009["Clusters"] = kmeans.labels_
alco2009.to_csv("Clustering_Result.csv", index=False)

data = alco2009[["Wine", "Beer"]]
label = alco2009["Clusters"]

predicted_result = kmeans.predict(data)
print("예측 클러스터링 결과")
print(predicted_result)

print("정답률: %s %%" %(metrics.accuracy_score(label,predicted_result)*100))

03. AI/01. Machine Learning/Cluster/01/clusters.py


# coding=utf-8
from __future__ import absolute_import

import time
import base64
import hashlib
import envs

from Crypto import Random
from Crypto.Cipher import AES

from inflection import underscore, pluralize
from mongokit import (Document, OR, ObjectId, Connection, Cursor,
                      DocumentMigration,
                      INDEX_DESCENDING, INDEX_ASCENDING)
from mongokit.document import DocumentProperties

# aliases
MongodbConn = Connection
ObjectId = ObjectId
INDEX_DESC = INDEX_DESCENDING
INDEX_ASC = INDEX_ASCENDING
Cursor = Cursor
DocumentMigration = DocumentMigration
OR = OR

class MetaDoc(DocumentProperties):

    def __init__(cls, name, bases, attrs):
        super(MetaDoc, cls).__init__(name, bases, attrs)
        if not name.startswith('Callable'):
            if '__collection__' not in cls.__dict__:
                cls.__collection__ = pluralize(underscore(cls.__name__))
        return

# models
class BaseDocument(Document):
    __metaclass__ = MetaDoc
    use_schemaless = False

    secret_key_prefix = envs.SECRET_KEY or '$b02NT~P=I$e*QZ'
    # migrate all encrypted data when change it.

    encrypt_secret_field = '_id'
    encrypt_mode = AES.MODE_ECB
    encrypt_block_size = AES.block_size

    sensitive_fields = []

    @staticmethod
    def _sensitive_filter(obj, repl='*'):
        if not dfa_filter.status:
            return obj
        elif isinstance(obj, basestring):
            return dfa_filter.filter(obj, repl)
        elif isinstance(obj, dict):
            res = {}
            for k, v in obj.iteritems():
                res[k] = dfa_filter.filter(v, repl)
            return res
        elif isinstance(obj, list):
            res = []
            for item in obj:
                res.append(dfa_filter.filter(item, repl))
            return res
        else:
            return obj

    def save(self, *args, **kwargs):
        for key in self.sensitive_fields:
            self[key] = self._sensitive_filter(self[key])
        if 'updated' in self:
            self['updated'] = int(time.time())
        return super(BaseDocument, self).save(*args, **kwargs)

    def write(self, *args, **kwargs):
        return super(BaseDocument, self).save(*args, **kwargs)

    # encryption (for unicode only)
    def encrypt(self, key, raw):
        if not raw:
            self[key] = raw
            return
        try:
            self[key] = self._encrypt(raw)
        except Exception as e:
            raise Exception('Document encrypt failed: {}'.format(str(e)))
        return self[key]

    def decrypt(self, key, key_prefix=None):
        if not self[key]:
            return self[key]
        try:
            return self._decrypt(self[key], key_prefix)
        except Exception:
            return self[key]

    def _encrypt(self, raw):
        if isinstance(raw, unicode):
            raw = raw.encode('utf-8')
        aes_key = self._get_aes_key()
        raw = self._pad(raw)
        iv = Random.new().read(self.encrypt_block_size)
        cipher = AES.new(aes_key, self.encrypt_mode, iv)
        return base64.b64encode(iv + cipher.encrypt(raw)).decode('utf-8')

    def _decrypt(self, enc, key_prefix=None):
        enc = base64.b64decode(enc)
        iv = enc[:self.encrypt_block_size]
        aes_key = self._get_aes_key(key_prefix)
        cipher = AES.new(aes_key, self.encrypt_mode, iv)
        return self._unpad(
            cipher.decrypt(enc[self.encrypt_block_size:])
        ).decode('utf-8')

    def _get_aes_key(self, key_prefix=None):
        key = self[self.encrypt_secret_field]
        _prefix = key_prefix or self.secret_key_prefix
        secret_key = u'{}.{}'.format(_prefix or u'', key)
        return hashlib.md5(secret_key).hexdigest()

    def _pad(self, s):
        _bs = self.encrypt_block_size
        return s + (_bs - len(s) % _bs) * chr(_bs - len(s) % _bs)

    def _unpad(self, s):
        return s[:-ord(s[len(s) - 1:])]

# filters
class DFAFilter():

    """Filter Messages from keywords
    Use DFA to keep algorithm perform constantly
    >>> f = DFAFilter()
    >>> f.add('fuck')
    >>> f.filter('hello fuck buddy')
    hello **** buddy
    """

    def __init__(self, path=None):
        self.keyword_chains = {}
        self.delimit = '\x00'
        if path:
            self.parse(path, True)

    @property
    def status(self):
        return bool(self.keyword_chains)

    def parse(self, path, required=True):
        try:
            with open(path) as f:
                for line in f:
                    self.add(line.strip())
        except Exception as e:
            if required:
                msg = 'DFAFilter: Dictionary text parse failed. '
                raise Exception(msg + str(e))

    def add(self, keyword):
        if not isinstance(keyword, unicode):
            keyword = keyword.decode('utf-8')
        chars = keyword.lower().strip()
        if not chars:
            return
        level = self.keyword_chains
        for i in xrange(len(chars)):
            if chars[i] in level:
                level = level[chars[i]]
            else:
                if not isinstance(level, dict):
                    break
                for j in xrange(i, len(chars)):
                    level[chars[j]] = {}
                    last_level, last_char = level, chars[j]
                    level = level[chars[j]]
                last_level[last_char] = {self.delimit: 0}
                break
        if i == len(chars) - 1:
            level[self.delimit] = 0

    def in_latin_word(self, message, start, step_ins):
        start_char = message[start]
        if 0x4e00 <= ord(start_char) < 0x9fa6:  # is chinese
            return False
        if start > 0 and \
           (ord(message[start - 1]) < 0x4e00 or
            ord(message[start - 1]) >= 0x9fa6) and \
           message[start - 1].isalnum():
            return True
        if start + step_ins < len(message) and \
           (ord(message[start + step_ins]) < 0x4e00 or
            ord(message[start + step_ins]) >= 0x9fa6) and \
           message[start + step_ins].isalnum():
            return True
        return False

    def filter(self, message, repl="*"):
        if not isinstance(message, str) and not isinstance(message, unicode):
            return message

        if isinstance(message, str):
            message = message.decode('utf-8')
        content_list = []
        start = 0
        while start < len(message):
            level = self.keyword_chains
            step_ins = 0
            for char in message[start:]:
                if char in level:
                    step_ins += 1
                    if self.delimit not in level[char]:
                        level = level[char]
                    else:
                        if self.in_latin_word(message, start, step_ins):
                            step_end = start + step_ins
                            content_list.append(message[start:step_end])
                            start += step_ins - 1
                            break
                        else:
                            content_list.append(repl * step_ins)
                            start += step_ins - 1
                            break
                else:
                    content_list.append(message[start])
                    break
            else:
                content_list.append(message[start])
            start += 1
        result = ''.join(content_list)
        return unicode(result)

    def check_legal(self, message, user_id):
        if not isinstance(message, unicode):
            message = message.decode('utf-8')
        message = message.lower()
        start = 0
        while start < len(message):
            level = self.keyword_chains
            step_ins = 0
            for char in message[start:]:
                if char in level:
                    step_ins += 1
                    if self.delimit not in level[char]:
                        level = level[char]
                    else:
                        return False
            start += 1
        return True

dfa_filter = DFAFilter('badwords.txt')


from __future__ import print_function

import os
""" First change the following directory link to where all input files do exist """
os.chdir("C:\\Users\\prata\\Documents\\book_codes\\NLP_DL")

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
import matplotlib.pyplot as plt
import nltk
import numpy as np

import pandas as pd
import random

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import string

from nltk import pos_tag
from nltk.stem import PorterStemmer

def preprocessing(text):
    text2 = " ".join("".join([" " if ch in string.punctuation else ch for ch in text]).split())

    tokens = [word for sent in nltk.sent_tokenize(text2) for word in
              nltk.word_tokenize(sent)]

    tokens = [word.lower() for word in tokens]

    stopwds = stopwords.words('english')
    tokens = [token for token in tokens if token not in stopwds]

    tokens = [word for word in tokens if len(word)>=3]

    stemmer = PorterStemmer()
    tokens = [stemmer.stem(word) for word in tokens]

    tagged_corpus = pos_tag(tokens)

    Noun_tags = ['NN','NNP','NNPS','NNS']
    Verb_tags = ['VB','VBD','VBG','VBN','VBP','VBZ']

    lemmatizer = WordNetLemmatizer()

    def prat_lemmatize(token,tag):
        if tag in Noun_tags:
            return lemmatizer.lemmatize(token,'n')
        elif tag in Verb_tags:
            return lemmatizer.lemmatize(token,'v')
        else:
            return lemmatizer.lemmatize(token,'n')

    pre_proc_text =  " ".join([prat_lemmatize(token,tag) for token,tag in tagged_corpus])

    return pre_proc_text

lines = []

fin = open("alice_in_wonderland.txt", "rb")

#fin = open("shakespeare.txt", "rb")

for line in fin:
    line = line.strip().decode("ascii", "ignore").encode("utf-8")

    if len(line) == 0:
        continue
    lines.append(preprocessing(line))
fin.close()

import collections
counter = collections.Counter()

for line in lines:
    for word in nltk.word_tokenize(line):
        counter[word.lower()]+=1

word2idx = {w:(i+1) for i,(w,_) in enumerate(counter.most_common())}
idx2word = {v:k for k,v in word2idx.items()}

xs = []
ys = []

for line in lines:
    embedding = [word2idx[w.lower()] for w in nltk.word_tokenize(line)]
    triples = list(nltk.trigrams(embedding))
    w_lefts = [x[0] for x in triples]
    w_centers = [x[1] for x in triples]
    w_rights = [x[2] for x in triples]
    xs.extend(w_centers)
    ys.extend(w_lefts)
    xs.extend(w_centers)
    ys.extend(w_rights)

print (len(word2idx))

vocab_size = len(word2idx)+1

ohe = OneHotEncoder(n_values=vocab_size)
X = ohe.fit_transform(np.array(xs).reshape(-1, 1)).todense()
Y = ohe.fit_transform(np.array(ys).reshape(-1, 1)).todense()
Xtrain, Xtest, Ytrain, Ytest,xstr,xsts = train_test_split(X, Y,xs, test_size=0.3,
                                                random_state=42)
print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape)

from keras.layers import Input,Dense,Dropout
from keras.models import Model

np.random.seed(42)

BATCH_SIZE = 128
NUM_EPOCHS = 20

input_layer = Input(shape = (Xtrain.shape[1],),name="input")
first_layer = Dense(300,activation='relu',name = "first")(input_layer)
first_dropout = Dropout(0.5,name="firstdout")(first_layer)

second_layer = Dense(2,activation='relu',name="second")(first_dropout)

third_layer = Dense(300,activation='relu',name="third")(second_layer)
third_dropout = Dropout(0.5,name="thirdout")(third_layer)

fourth_layer = Dense(Ytrain.shape[1],activation='softmax',name = "fourth")(third_dropout)

history = Model(input_layer,fourth_layer)
history.compile(optimizer = "rmsprop",loss="categorical_crossentropy",metrics=["accuracy"])

history.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE,epochs=NUM_EPOCHS, verbose=1,validation_split = 0.2)

# Extracting Encoder section of the Model for prediction of latent variables
encoder = Model(history.input,history.get_layer("second").output)

# Predicting latent variables with extracted Encoder model
reduced_X = encoder.predict(Xtest)

final_pdframe = pd.DataFrame(reduced_X)
final_pdframe.columns = ["xaxis","yaxis"]
final_pdframe["word_indx"] = xsts
final_pdframe["word"] = final_pdframe["word_indx"].map(idx2word)

rows = random.sample(final_pdframe.index, 100)
vis_df = final_pdframe.ix[rows]

labels = list(vis_df["word"]);xvals = list(vis_df["xaxis"])
yvals = list(vis_df["yaxis"])

#in inches
plt.figure(figsize=(10, 10))

for i, label in enumerate(labels):
    x = xvals[i]
    y = yvals[i]
    plt.scatter(x, y)
    plt.annotate(label,xy=(x, y),xytext=(5, 2),textcoords='offset points',
                 ha='right',va='bottom')

plt.xlabel("Dimension 1")
plt.ylabel("Dimension 2")
plt.show()


#!/usr/bin/python
# -*- coding: utf-8 -*-
## @file processing.py
# @brief Файл содержит основные функции генерации графики

import astrocv.castrocv as acv
from astrocv.zernike import *

## @brief - Функция добавления шума
# @param img - исходное изображение
# @param background - средний фон
# @param dNoise - дельта
# @param bpp - bits per pixel
# @param Stride - stride изображения
# @return - Вернет изображение
def addNoiseUniform(img, background, dNoise):
    bpp = int(img.nbytes / img.size) * 8
    if bpp != 8 and bpp != 16 and bpp != 32:
        raise Exception("Unsupported pixel type {}".format(img.dtype))
    Stride = len(img[0]) * (bpp // 8)
    w = len(img[0])
    h = len(img)
    params = [img.data, w, Stride, h, int(background), int(-dNoise), int(+dNoise), bpp]
    w, h = acv.addNoiseUniform(*params)
    return img

## @brief - Функция добавления шума с нормальным распределением
# @param img - исходное изображение
# @param M - математическое ожидание
# @param S - сигма
# @param bpp - bits per pixel
# @param Stride - stride изображения
# @return - Вернет изображение
def addNoiseNorm(img, M, S):
    bpp = int(img.nbytes / img.size) * 8
    if bpp != 8 and bpp != 16 and bpp != 32:
        raise Exception("Unsupported pixel type {}".format(img.dtype))
    Stride = len(img[0]) * (bpp // 8)
    w = len(img[0])
    h = len(img)
    params = [img.data, w, Stride, h, M, S, bpp]
    w, h = acv.addNoiseNorm(*params)
    return img

## @brief - Функция рисования закрашенного круга с распределением яркости по косинусу
# @param img - исходное изображение
# @param X - координата X
# @param Y - координата Y
# @param Diameter - диаметр круга
# @param bpp - bits per pixel
# @param Stride - stride изображения
# @param Signal - яркость круга
# @return - Вернет изображение
def drawObject(img, X, Y, Diameter, Signal=None):
    bpp = int(img.nbytes / img.size) * 8
    if bpp != 8 and bpp != 16 and bpp != 32:
        raise Exception("Unsupported pixel type {}".format(img.dtype))
    Stride = len(img[0]) * (bpp // 8)
    if Signal is None:
        Signal = (1 << bpp) - 1
        if bpp == 32:
            Signal //= 2
    w = len(img[0])
    h = len(img)
    params = [img.data, w, Stride, h, X, Y, Signal, int(Diameter/2), bpp]
    return acv.drawObject(*params)

## @brief - Функция рисования закрашенного круга с распределением яркости по косинусу
# @param img - исходное изображение
# @param X - координата X
# @param Y - координата Y
# @param Diameter - диаметр круга
# @param bpp - bits per pixel
# @param Stride - stride изображения
# @param Signal - яркость круга
# @return - Вернет изображение
def drawObjectPolinomials(img, X, Y, coefficients, power=1e-8, diamAper=0.2, blindZone=0.1,
                          wavelength=0.5e-6, pixSize=5e-6, focus=2.0,exposition=1.0, electronWell=8e3, quantumEffect=1.0):
    bpp = int(img.nbytes / img.size) * 8
    if bpp != 8 and bpp != 16 and bpp != 32:
        raise Exception("Unsupported pixel type {}".format(img.dtype))
    w = len(img[0])
    h = len(img)
    offsetX, offsetY = coefficients[1], coefficients[2]
    coefficients[1] = coefficients[2] = 0
    size = 200
    ampl, poli = makeNearZone(size, coefficients, power, blindZone)
    far = makeFarZone(ampl, poli, diamAper, wavelength, pixSize, focus, exposition, electronWell, quantumEffect)

    left = int(X-len(far)/2)
    right = int(X+len(far)/2)
    top = int(Y - len(far)/2)
    bottom = int(Y + len(far) / 2)
    if left < 0:
        left = 0
    if right > w:
        right = w
    if top < 0:
        top = 0
    if bottom > h:
        bottom = h

    for i in range(0, len(far)):
        img[top + i][left:right] += far[i][0:len(far)]

    return 0

## @brief - Функция выделения объекта на изображении
# @param img - исходное изображение
# @param X - координата X
# @param Y - координата Y
# @param Left - координата левой границы объекта
# @param Top - координата верхней границы объекта
# @param Right - координата правой границы объекта
# @param Bottom - координата нижней границы объекта
# @return - Вернет изображение
def addMarker(img, X, Y, Left, Top, Right, Bottom):
    bpp = int(img.nbytes / img.size) * 8
    if bpp != 8 and bpp != 16 and bpp != 32:
        raise Exception("Unsupported pixel type {}".format(img.dtype))
    Stride = len(img[0]) * (bpp // 8)
    w = len(img[0])
    h = len(img)
    params = [img.data, w, Stride, h, int(X), int(Y), Left, Top, Right, Bottom, bpp]
    w, h = acv.addMarker(*params)
    return img

if __name__ == '__main__':
    import cv2
    import numpy as np

    width = 800
    height = 600

    Noise = 30
    dNoise = 7
    bpp = 8
    type = {8: np.uint8, 16: np.uint16, 32: np.int32}[bpp]
    white_level = (1 << 8) - 1
    if bpp != 8:
        white_level = (1 << 16) - 1

    while True:
        img = np.zeros((height, width), dtype=type)
        #addNoiseNorm(img, Noise, dNoise)
        K = [np.random.random() for i in range(15)]
        #K = [1.0, 0.0, 0.0, 0.1, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0]

        drawObjectPolinomials(img, int(width/2), int(height/2), K, exposition=2)

        cv2.imshow('img', img)

        val = cv2.waitKey(1)
        if val & 0xff == ord('q'):
            exit(0)


"""Unit test for posydon.active_learning.psy_cris classes
"""
import unittest
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import math

from posydon.active_learning.psy_cris.data import TableData
from posydon.active_learning.psy_cris.regress import Regressor
from posydon.active_learning.psy_cris.classify import Classifier
from posydon.active_learning.psy_cris.sample import Sampler

from posydon.active_learning.psy_cris.utils import (
    get_regular_grid_df,
    get_random_grid_df,
)
from posydon.active_learning.psy_cris.synthetic_data.synth_data_3D import get_output_3D

SKIP_TEST_PLOTS = False
SHOW_PLOTS = False

class TestSampler(unittest.TestCase):
    """Test Sampler class on the 3d synthetic data set."""

    @classmethod
    def setUpClass(cls):
        np.random.seed(12345)
        cls.TEST_DATA_GRID = get_regular_grid_df(N=10 ** 3, dim=3)
        cls.TEST_DATA_RAND = get_random_grid_df(N=10 ** 3, dim=3)
        cls.UNIQUE_CLASSES = [1, 2, 3, 4, 6, 8]

        cls.TEST_INPUT_POINTS = np.array([[0, 0, 0], [-0.5, 0.5, 0.5]])
        cls.TEST_OUTPUT_TRUTH = get_output_3D(*cls.TEST_INPUT_POINTS.T)

    def setUp(self):
        my_kwargs = {"n_neighbors": [2, 3, 5]}
        self.table_grid = self.create_TableData(self.TEST_DATA_GRID, **my_kwargs)
        self.regr_grid = Regressor(self.table_grid)
        self.cls_grid = Classifier(self.table_grid)

    def create_TableData(self, data_frame, **kwargs):
        files = None
        input_cols = ["input_1", "input_2", "input_3"]
        output_cols = ["class", "output_1"]
        class_col_name = "class"
        table_obj = TableData(
            files,
            input_cols,
            output_cols,
            class_col_name,
            my_DataFrame=data_frame,
            verbose=False,
            **kwargs
        )
        return table_obj

    def train_everything_grid(self, cls_names, regr_names):
        if cls_names is not None:
            self.cls_grid.train_everything(cls_names)
        if regr_names is not None:
            self.regr_grid.train_everything(regr_names)

    def test_init_0(self):
        test_cases = [
            (None, None),
            (self.cls_grid, self.regr_grid),
            (None, self.regr_grid),
        ]
        for i, tup_input in enumerate([(None, None), ()]):
            with self.subTest("Sampler init", iter=i):
                samp = Sampler(*tup_input)

    def test_mcmc(self):
        self.train_everything_grid(["rbf"], None)
        samp = Sampler(classifier=self.cls_grid, regressor=None)

        steps, acc, rej = samp.run_MCMC(
            15, 0.25, [0, 0, 0], samp.TD_classification, "rbf", T=1, **{"TD_BETA": 2}
        )
        self.assertTrue(len(steps) == (acc + 1), msg="steps taken should match acc.")

    def test_ptmcmc(self):
        self.train_everything_grid(["rbf"], ["rbf"])
        samp = Sampler(classifier=self.cls_grid, regressor=self.regr_grid)

        chain_step_hist, T_list = samp.run_PTMCMC(
            5,
            15,
            samp.TD_classification_regression,
            ("rbf", "rbf"),
            init_pos=[0, 0, 0],
            alpha=0.25,
            verbose=False,
            trace_plots=False,
            TD_BETA=1,
        )
        # try with default values
        chain_step_hist, T_list = samp.run_PTMCMC(
            10, 15, samp.TD_classification_regression, ("rbf", "rbf"),
            verbose=False, trace_plots=False)

    def test_simple_density_logic(self):
        self.cls_grid.train("rbf")
        samp = Sampler(classifier=self.cls_grid, regressor=None)
        steps, acc, rej = samp.run_MCMC(
            200, 0.25, [0, 0, 0], samp.TD_classification, "rbf", T=1
        )
        acc_pts, rej_pts = samp.do_simple_density_logic(steps, 10, 0.05)
        return samp, steps

    def test_get_proposed_points(self):
        N = 10
        samp, step_hist = self.test_simple_density_logic()
        prop_points, kappa = samp.get_proposed_points(step_hist, N, 0.046)
        self.assertTrue(len(prop_points) == N)

    @unittest.skipIf(SKIP_TEST_PLOTS, "Plotting C, C+R target distributions")
    def test_TD_plots(self):
        self.train_everything_grid(["rbf"], ["rbf"])
        samp = Sampler(classifier=self.cls_grid, regressor=self.regr_grid)

        N = 70 if SHOW_PLOTS else 5
        zed = 0
        x, y = np.meshgrid(np.linspace(-1, 1, N), np.linspace(-1, 1, N))
        z = np.ones(x.shape) * zed
        data_points = np.concatenate(
            (x.flatten()[:, None], y.flatten()[:, None], z.flatten()[:, None]), axis=1
        )

        max_probs, pos, cls_keys = samp.get_TD_classification_data("rbf", data_points)

        kwargs = {"TD_BETA": 2, "TD_TAU": 0.5}
        cls_regr_td_vals = [
            float(samp.TD_classification_regression(["rbf", "rbf"], dat, **kwargs))
            for dat in data_points
        ]

        fig, subs = plt.subplots(1, 2, figsize=(13, 5))
        subs[0].set_title("TD_classification at z = {}".format(zed))
        cls_plot = subs[0].pcolormesh(
            x, y, (1 - max_probs).reshape(N, N), shading="auto"
        )

        subs[1].set_title("TD_classification_regression at z = {}".format(zed))
        cls_regr_plot = subs[1].pcolormesh(
            x, y, np.array(cls_regr_td_vals).reshape(N, N), shading="auto"
        )

        fig.colorbar(cls_plot, ax=subs[0])
        fig.colorbar(cls_regr_plot, ax=subs[1])

        for i in range(2):
            subs[i].set_xlabel("input_1")
            subs[1].set_ylabel("input_2")
            subs[i].axis("equal")

        if SHOW_PLOTS:
            plt.show()
        plt.close()

if __name__ == "__main__":
    unittest.main()


# https://leetcode.com/problems/merge-strings-alternately/

class Solution:
    def mergeAlternately(self, word1: str, word2: str) -> str:
        len1 = len(word1)
        len2 = len(word2)
        low = min(len1, len2)
        remaining = abs(len1 - len2)
        new_word = []
        for i in range(low):
            new_word.append(word1[i])
            new_word.append(word2[i])
        if remaining:
            if len1 > len2:
                leftover = word1
            else:
                leftover = word2
            for j in range(remaining):
                new_word.append(leftover[low+j])
        ret_word = "".join(str(element) for element in new_word)
        return ret_word

rahulrana/python/Leetcode/#058 Merge Strings Alternately/v58.0.py


import math

def getPermutation(n, k):
    templist = []  # 定义一个列表，用于装放下面取出的数
    temp = [i for i in range(1, n + 1)]  # 定义一个n位的列表1-n
    dicts = {1: 1, 2: 1, 3: 2, 4: 6, 5: 24, 6: 120, 7: 720, 8: 5040, 9: 40320, 10: 362880}  # 定义各位的权值#
    for i in range(n, 0, -1):
        s = math.ceil(k / dicts[i])  # 计算商值 取其天花
        templist.append(temp.pop(s - 1))  # 将对应数值，加入到templist中，并删除在temp中取出得数，避免重复取出
        if k % dicts[i] == 0:  # 判断能整除
            templist.extend(temp[::-1])  # 将剩余数反转，即最大组合数
            break
        else:  # 如果不能整除，即将k值等于其余数
            k %= dicts[i]
    return ''.join([str(i) for i in templist])  # 将列表中数字先转化为str`

n = int(input())
k = int(input())
print(getPermutation(n, k))


#! /usr/bin/env python
# -*- coding: utf-8 -*-
from django.conf.urls import url, include
from monitor import system, manage, api

urlpatterns = [
    url(r'^system/$', system.index, name='monitor'),
    url(r'^manage/delall/$', manage.drop_sys_info, name='drop_all'),
    url(r'^hosttree/$', system.tree_node, name='host_tree'),
    url(r'^manage/delrange/(?P<timing>[0-9])/$', manage.del_monitor_data, name='del_monitor_data'),
    url(r'^manage/$', manage.index, name='monitor_manage'),
    url(r'^system/(?P<hostname>.+)/(?P<timing>\d+)/$', system.host_info, name='host_info'),
    url(r'^getcpu/(?P<hostname>.+)/(?P<timing>\d+)/$', system.get_cpu, name='get_cpu'),
    url(r'^getmem/(?P<hostname>.+)/(?P<timing>\d+)/$', system.get_mem, name='get_mem'),
    url(r'^getdisk/(?P<hostname>.+)/(?P<timing>\d+)/(?P<partition>\d+)/$', system.get_disk, name='get_disk'),
    url(r'^getnet/(?P<hostname>.+)/(?P<timing>\d+)/(?P<net_id>\d+)/$', system.get_net, name='get_net'),
    url(r'^received/sys/info/$', api.received_sys_info, name='received_sys_info'),
]


import sys
import re
from docx import Document

pmid_re = re.compile('PMID: (\d*)')

def process_document(document: Document):
    pmids = set()
    for para in document.paragraphs:
        text = para.text

        re_results = pmid_re.search(text)
        if re_results is None:
            continue

        pmid_groups = re_results.groups()
        if pmid_groups:
            for x in pmid_groups:
                pmids.add(int(x))

    for pmid in sorted(pmids):
        print(pmid)

if __name__ == '__main__':
    doc_file = sys.argv[1]
    process_document(Document(doc_file))


from flask import Flask, render_template, url_for, request
import numpy as np
from sklearn.externals import joblib
app = Flask(__name__)

filename = './data/model.sav'
model = joblib.load(filename)

def find_species(a,b,c,d):
	return model.predict(np.array([a,b,c,d]).reshape(1,-1))[0]

@app.route("/")
@app.route("/index.html")
def first_index():
	return render_template('index.html', data=' ')

@app.route("/", methods=['POST'])
def second_index():
	sepall = request.form['sepall']
	sepalw = request.form['sepalw']
	petall = request.form['petall']
	petalw = request.form['petalw']
	species = find_species(sepall, sepalw, petall, petalw)
	return render_template('index.html', data=species)

IRIS Flower Classification Project/main.py


from flask import url_for, request, jsonify
import requests
import ssl
from datetime import datetime
import time
import os
backend_url = os.environ.get('backend_url')
google_api_key = os.environ.get('google_api_key')
if backend_url is None:
    backend_url = 'https://kinhkcovid19dataengine.herokuapp.com/'

def update_building_geo_sc():
    print("update_building_geo start :" + time.strftime("%A, %d. %B %Y %I:%M:%S %p"))
    url_GetXY_Pre = "https://nominatim.openstreetmap.org/search?q="
    url_GetXY_Post = "&format=json&polygon=1&addressdetails=1"

    url_GetXY_Pre_google ="https://maps.googleapis.com/maps/api/geocode/json?address="
    url_GetXY_Post_google = ",Hong Kong&key=" + google_api_key

    header = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest"
    }
    data_obj = []
    response_building = requests.get(backend_url + "getBuildingWithoutGeoData",data = data_obj, headers=header)
    response_building_json_data = response_building.json()
    inserted = 0
    print("No Geo Data Building : " +str(len(response_building_json_data)))
    for building_data in response_building_json_data:
        district = building_data["district"]
        building_name = building_data["building_name"]
        try:
            map_request_starttime = datetime.now()
            response = requests.get(url_GetXY_Pre+building_name+","+district+url_GetXY_Post,data = data_obj, headers=header)
            json_data = response.json()
            if len(json_data) > 0:
                for data in json_data:
                    new_data_obj = {"district":district,
                        "building_name":building_name,
                        "lat": data["lat"],
                        "lon": data["lon"]}
                    response = requests.post(backend_url + "/saveBuildingGEO",data = new_data_obj, headers=header)
                    inserted+=1
                    print('Updated by Open Street:' + str(inserted))
                    break
            else:
                print("No result from Open Street Map - Go for Google")
                urlToGoogle = url_GetXY_Pre_google+building_name+","+district+url_GetXY_Post_google
                response = requests.get(urlToGoogle,data = data_obj, headers=header)
                json_data = response.json()
                result_data = json_data["results"]
                if len(result_data) > 0:
                    for data in result_data:
                        geometry = data["geometry"]
                        location = geometry["location"]
                        new_data_obj = {"district":district,
                        "building_name":building_name,
                        "lat": location["lat"],
                        "lon": location["lng"]}
                        response = requests.post(backend_url + "/saveBuildingGEO",data = new_data_obj, headers=header)
                        inserted+=1
                        print('Updated by Google:' + str(inserted))
                        break
                else:
                    print("No result from Open Street Map & Google " + building_name +","+district)
                    new_data_obj = {"district":district,
                    "building_name":building_name}
                    response = requests.post(backend_url + "/saveEmptyBuildingGEO",data = new_data_obj, headers=header)
                    inserted+=1
                    print('Updated Empty GEO:' + str(inserted))

        except:
             print('Request except')
        if inserted >= 150:
            break
        else:
            map_request_endtime = datetime.now()
            difference = (map_request_endtime - map_request_starttime)
            total_seconds = difference.total_seconds()
            if total_seconds < 1:
                time.sleep(1)
    print("GEO Data added :" + str(inserted))
    print("update_building_geo end :" + time.strftime("%A, %d. %B %Y %I:%M:%S %p"))

update_building_geo_sc()


from flask import Flask, render_template, flash, redirect, render_template
from flask_debugtoolbar import DebugToolbarExtension
from models import db, connect_db, Pet
from forms import AddPetForm, EditPetForm

app = Flask(__name__)
app.config["SECRET_KEY"] = "oh-so-secret"
app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql:///adopt"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False

debug = DebugToolbarExtension(app)

connect_db(app)
db.create_all()

@app.route("/")
def homepage():
    """Show homepage"""

    pets = Pet.query.all()
    return render_template("index.html", pets=pets)

@app.route("/add", methods=["GET", "POST"])
def add_pet():
    """Add a pet."""

    form = AddPetForm()

    if form.validate_on_submit():
        data = {k: v for k, v in form.data.items() if k != "csrf_token"}
        new_pet = Pet(**data)
        db.session.add(new_pet)
        db.session.commit()
        flash(f"{new_pet.name} added.")
        return redirect('/')

    else:
        # re-present form for editing
        return render_template("form.html", form=form)

@app.route("/<int:id>", methods=["GET", "POST"])
def edit_pet(id):
    """Edit pet."""

    pet = Pet.query.get_or_404(id)
    form = EditPetForm(obj=pet)

    if form.validate_on_submit():
        pet.notes = form.notes.data
        pet.available = form.available.data
        pet.photo_url = form.photo_url.data
        db.session.commit()
        flash(f"{pet.name} updated.")
        return redirect('/')

    else:
        # failed; re-present form for editing
        return render_template("edit.html", form=form, pet=pet)


import sys
import os
import json
import pickle
import torch
import cv2

current_dir = os.getcwd()
choose_frames_all_dir = os.path.join(current_dir, "choose_frames_all")
model = torch.hub.load('WongKinYiu/yolov7', 'custom', 'yolov7.pt',
                       force_reload=True, trust_repo=True)

avaMin_dense_proposals_path = os.path.join(
    current_dir, "dense_proposal_train.pkl")

results_dict = {}
dicts = []

for root, dirs, files in os.walk(choose_frames_all_dir):
    lenFile = len(files)/3
    files.sort(key=lambda arr: (int(arr[:-7]), int(arr[3:-4])))
    for name in files:
        temp_file_name = name.split("_")[0]
        temp_video_ID = name.split("_")[1].split('.')[0]
        temp_video_ID = int(temp_video_ID)
        temp_video_ID = str(int((temp_video_ID-1)/30))
        temp_video_ID = temp_video_ID.zfill(4)

        if int(temp_video_ID) <= 1 or int(temp_video_ID) >= lenFile:
            continue

        key = temp_file_name + ',' + temp_video_ID
        frame_path = os.path.join(root, name)
        frame = cv2.imread(frame_path)
        model_results = model(frame)
        model_results.pandas().xyxy[0]
        objects = model_results.pandas().xyxy[0].to_json(orient="records")
        objects = json.loads(objects)
        results = []

        for object in objects:
            parsed = object
            # if parsed['confidence'] > gun_model.conf:
            if parsed['name'] == 'car' or parsed['name'] == 'person':
                top = int(parsed['ymin'])
                right = int(parsed['xmax'])
                bottom = int(parsed['ymax'])
                left = int(parsed['xmin'])
                results.append(
                    [left, top, right, bottom, parsed['confidence']])
                dicts.append([left, top, right, bottom, parsed['confidence']])
        results_dict[key] = results
with open(avaMin_dense_proposals_path, "w") as pklfile:
    pickle.dump(results_dict, pklfile)

for i in results_dict:
    print(i, results_dict[i])