# -*- coding: utf-8 -*-
# GNU General Public License v2.0 (see COPYING or https://www.gnu.org/licenses/gpl-2.0.txt)

from __future__ import absolute_import, division, unicode_literals

import json
import timeit

import constants
import file_utils
import image_utils
import utils
import xbmc
from settings import SETTINGS

try:
    from queue import (
        Queue,
        Empty as QueueEmpty,
        Full as QueueFull,
    )
except ImportError:
    from Queue import (
        Queue,
        Empty as QueueEmpty,
        Full as QueueFull,
    )

class UpNextHashStore(object):
    """Class to store/save/load hashes used by UpNextDetector"""

    __slots__ = (
        'version',
        'hash_size',
        'group_name',
        'group_idx',
        'data',
        'timestamps'
    )

    def __init__(self, **kwargs):
        self.version = kwargs.get('version', 0.2)
        self.hash_size = kwargs.get('hash_size', (8, 8))
        item = kwargs.get('item', {})
        self.group_name = item.get('group_name', '')
        self.group_idx = item.get('group_idx') or constants.UNDEFINED
        self.data = kwargs.get('data', {})
        self.timestamps = kwargs.get('timestamps', {self.group_idx: None})

    @staticmethod
    def int_to_hash(val, hash_size):
        return tuple([  # pylint: disable=consider-using-generator
            1 if bit_val == "1" else 0
            for bit_val in bin(val)[2:].zfill(hash_size)
        ])

    @staticmethod
    def hash_to_int(image_hash):
        return sum(
            (bit_val or 0) << i
            for i, bit_val in enumerate(reversed(image_hash))
        )

    @classmethod
    def log(cls, msg, level=utils.LOGDEBUG):
        utils.log(msg, name=cls.__name__, level=level)

    def is_valid(self, item=None, for_saving=False):

        if item:
            group_name = item.get('group_name')
            group_idx = item.get('group_idx')
        else:
            group_name = None
            group_idx = None

        # Non-episodic video is being played
        if not self.group_name or self.group_idx == constants.UNDEFINED:
            return False

        # Playlist with no episode details
        if for_saving and self.group_name == constants.MIXED_PLAYLIST:
            return False

        # No new episode details, assume current hashes are still valid
        if group_name is None and group_idx is None:
            return True

        # Current episode matches, current hashes are still valid
        if self.group_name == group_name and self.group_idx == group_idx:
            return True

        # New video is being played, invalidate old hashes
        return False

    def invalidate(self):
        self.group_name = ''
        self.group_idx = constants.UNDEFINED

    def load(self, identifier):
        target = file_utils.get_legal_filename(
            identifier, prefix=SETTINGS.detector_save_path, suffix='.json'
        )
        try:
            with open(target, mode='r', encoding='utf-8') as target_file:
                hashes = json.load(target_file)
        except (IOError, OSError, TypeError, ValueError):
            self.log('Could not load stored hashes from {0}'.format(target))
            return False

        if not hashes:
            return False

        self.version = float(hashes.get('version', self.version))
        self.hash_size = hashes.get('hash_size', self.hash_size)
        if 'data' in hashes:
            hash_size = self.hash_size[0] * self.hash_size[1]
            self.data = {
                tuple([utils.get_int(i) for i in key[1:-1].split(', ')]):  # pylint: disable=consider-using-generator
                self.int_to_hash(hashes['data'][key], hash_size)
                for key in hashes['data']
            }
        if 'timestamps' in hashes:
            self.timestamps = {
                utils.get_int(group_idx):
                    hashes['timestamps'][group_idx]
                for group_idx in hashes['timestamps']
                if group_idx != 'null'
            }

        self.log('Hashes loaded from {0}'.format(target))
        return True

    def save(self, identifier):
        output = {
            'version': self.version,
            'hash_size': self.hash_size,
            'data': {
                str(hash_index): self.hash_to_int(self.data[hash_index])
                for hash_index in self.data
                if hash_index[-1] != constants.UNDEFINED
            },
            'timestamps': self.timestamps
        }

        target = file_utils.get_legal_filename(
            identifier, prefix=SETTINGS.detector_save_path, suffix='.json'
        )
        try:
            with open(target, mode='w', encoding='utf-8') as target_file:
                json.dump(output, target_file, indent=4)
                self.log('Hashes saved to {0}'.format(target))
        except (IOError, OSError, TypeError, ValueError):
            self.log('Could not save hashes to {0}'.format(target),
                     utils.LOGWARNING)
        return output

    def window(self, hash_index,
               size=SETTINGS.detect_matches, all_episodes=False):
        """Get sets of hashes, either from all episodes or only from the first
        and last episodes, where the timestamps are approximately equal (+/- an
        adjustable offset) to the timestamps of the reference hash index"""

        end_time, start_time, episode = hash_index

        if all_episodes:
            excluded_episodes = [constants.UNDEFINED]
            selected_episodes = self.timestamps.keys()
        else:
            excluded_episodes = [constants.UNDEFINED, episode]
            selected_episodes = {min(self.timestamps), max(self.timestamps)}

        # Matching time period from start of file
        min_start_time = start_time - size
        max_start_time = start_time + size
        # Matching time period from end of file
        min_end_time = end_time - size
        max_end_time = end_time + size

        return {
            hash_index: self.data[hash_index]
            for hash_index in self.data
            if hash_index[2] in selected_episodes
            and hash_index[2] not in excluded_episodes
            and (
                min_start_time <= hash_index[1] <= max_start_time
                or min_end_time <= hash_index[0] <= max_end_time
            )
        }

class UpNextDetector(object):
    """Detector class used to detect end credits in playing video"""

    __slots__ = (
        # Instances
        'hashes',
        'past_hashes',
        'player',
        'state',
        # Settings
        'match_number',
        'mismatch_number',
        # Variables
        'capture_interval',
        'hash_index',
        'match_counts',
        # Worker pool
        'queue',
        'workers',
        # Signals
        '_lock',
        '_running',
        '_sigstop',
        '_sigterm'
    )

    def __init__(self, player, state):
        self.log('Init')

        self.player = player
        self.state = state
        self.queue = None
        self.workers = None

        self.match_counts = {
            'hits': 0,
            'misses': 0,
            'detected': False
        }
        self._lock = utils.create_lock()
        self._init_hashes()

        self._running = utils.create_event()
        self._sigstop = utils.create_event()
        self._sigterm = utils.create_event()

    @staticmethod
    def _and(bit1, bit2):
        return 1 if (bit1 and bit2) else 0

    @staticmethod
    def _eq_biased(bit1, bit2):
        return (bit1 == bit2) * (1 if bit2 else 0.5)

    @staticmethod
    def _mul(bit1, bit2):
        return bit1 * bit2

    @staticmethod
    def _xor(bit1, bit2):
        # bit1 and bit2 are bools
        # bit1 may be None
        return 0 if bit1 is None or bit1 == bit2 else 1

    @staticmethod
    def _generate_initial_hash(hash_width, hash_height, **kwargs):
        blank_token = (0, )
        pixel_token = (1, )
        border_token = (0, )
        ignore_token = (None, )

        pad_height = kwargs.get('pad_height', 0)
        pad_width = kwargs.get('pad_width', 3)
        pad_width_alt = kwargs.get('pad_width_alt', pad_width - 1)
        fuzz_height = kwargs.get('fuzz_height', 1)

        pad_width = (pad_width * hash_width // 16) - (hash_width // 16)
        pad_width_alt = (pad_width_alt * hash_width // 16) - (hash_width // 16)

        return (
            border_token * hash_width * pad_height
            + (
                border_token
                + blank_token * 2 * pad_width
                + ignore_token * (hash_width - 4 * pad_width - 2)
                + blank_token * 2 * pad_width
                + border_token
            ) * fuzz_height
            + ((
                border_token
                + blank_token * pad_width
                + ignore_token * pad_width
                + pixel_token * (hash_width - 4 * pad_width - 2)
                + ignore_token * pad_width
                + blank_token * pad_width
                + border_token
            ) + (
                border_token
                + blank_token * pad_width_alt
                + ignore_token * pad_width_alt
                + pixel_token * (hash_width - 4 * pad_width_alt - 2)
                + ignore_token * pad_width_alt
                + blank_token * pad_width_alt
                + border_token
            )) * ((hash_height - 2 * pad_height - 2 * fuzz_height) // 2)
            + (
                border_token
                + blank_token * 2 * pad_width
                + ignore_token * (hash_width - 4 * pad_width - 2)
                + blank_token * 2 * pad_width
                + border_token
            ) * fuzz_height
            + border_token * hash_width * pad_height
        )

    @staticmethod
    def _generate_mask(image_hash):
        fuzzy_value = None
        masked_value = 0

        mask = len(image_hash) / image_hash.count(masked_value)
        fuzzy_mask = 0.25
        min_mask = 0.25

        return tuple(
            mask if bit == masked_value else
            fuzzy_mask if bit == fuzzy_value else
            min_mask
            for bit in image_hash
        )

    @staticmethod
    def _create_hash(image, hash_size, output_file=None):
        image_hash = image_utils.process(
            image,
            queue=[
                [image_utils.resize, hash_size],
                [image_utils.points_of_interest],
                [image_utils.export_data],
            ],
            save_file=output_file
        )

        return image_hash

    @classmethod
    def _create_images(cls, image_data, image_size):
        image = image_utils.process(
            image_data,
            queue=[
                [image_utils.import_data, image_size, False],
                [image_utils.resize, cls._get_video_capture_resolution()],
                [image_utils.saturation],
                [image_utils.auto_level, 5, 95, (0.33, None)],
            ],
            save_file='1_image'
        )

        filtered_image = image_utils.process(
            image,
            queue=[
                [image_utils.posterise, 3],
                [image_utils.adaptive_filter, (8, 1, True),
                 image_utils.auto_level, (5, 95, (0.33, None))],
                [image_utils.apply_filter,
                 'UnsharpMask,20,400,64', 'TRIM'],
                [image_utils.apply_filter,
                 'RankFilter,5,50', 'TRIM', None, 'difference'],
                [image_utils.detail_reduce, image, 50],
                [image_utils.apply_filter,
                 'GaussianBlur,5', 'TRIM', None, 'multiply'],
                [image_utils.auto_threshold],
            ],
            save_file='2_filter'
        ) if SETTINGS.detector_filter else image

        return image, filtered_image

    @classmethod
    def _hash_fuzz(cls, image_hash, masking_hash, factor=5):
        weights = cls._generate_mask(masking_hash)

        significant_bits = sum(map(cls._mul, image_hash, weights))
        significance = 100 * significant_bits / len(image_hash)
        delta = significance - SETTINGS.detect_significance

        return factor * delta / SETTINGS.detect_significance

    @classmethod
    def _hash_similarity(cls, baseline_hash, image_hash, filtered_hash=None):
        """Method to compare the similarity between image hashes"""

        # Check that hashes are not empty and that dimensions are equal
        if not baseline_hash or not image_hash:
            return 0

        compare_hash = filtered_hash or image_hash

        num_pixels = len(baseline_hash)
        if num_pixels != len(compare_hash):
            return 0

        # Check whether each pixel is equal
        bits_eq = sum(map(cls._eq_biased, baseline_hash, compare_hash))
        bits_xor = tuple(map(cls._xor, baseline_hash, compare_hash))
        bits_xor_baseline = sum(map(cls._and, bits_xor, baseline_hash))
        bits_xor_compare = sum(map(cls._and, bits_xor, compare_hash))

        weighted_total = (
            num_pixels
            - baseline_hash.count(None)
            - (min(baseline_hash.count(0), compare_hash.count(0)) / 2)
        )
        bit_compare = bits_eq - bits_xor_baseline - bits_xor_compare

        # Evaluate similarity as a percentage of un-ignored pixels in the hash
        similarity = max(0, 100 * bit_compare / weighted_total)

        if not filtered_hash:
            uncertainty = 0
        elif filtered_hash != image_hash:
            uncertainty = cls._hash_fuzz(image_hash, filtered_hash)
        else:
            uncertainty = cls._hash_fuzz(image_hash, baseline_hash)

        return similarity - uncertainty

    @classmethod
    def _get_video_capture_resolution(cls, max_size=8):
        """Method to return a scaled down capture resolution tuple for use in
           capturing the video frame buffer at a specific size/resolution"""

        width, height, aspect_ratio = cls.get_video_resolution()

        # Capturing render buffer at higher resolution captures more detail
        # depending on Kodi scaling function used, but slows down processing.
        # Limit captured data to max_size (in kB)
        if max_size:
            max_size = max_size * 8 * 1024
            height = min(int((max_size / aspect_ratio) ** 0.5), height)
            width = min(int(height * aspect_ratio), width)

        return width, height

    @classmethod
    def _print_hashes(cls, hashes, size, prefix=''):
        """Method to print image hashes, side by side, to the Kodi log"""

        if not hashes:
            return

        num_bits = size[0] * size[1]
        row_length = size[0]

        hashes = [image_hash if image_hash and len(image_hash) == num_bits
                  else (0, ) * num_bits
                  for image_hash in hashes]

        cls.log('\n\t\t\t'.join([
            prefix,
            '{0}|{1}|'.format(
                size,
                '|'.join(str(UpNextHashStore.hash_to_int(image_hash))
                         for image_hash in hashes)
            )
        ] + ['{0:>3}|{1}|'.format(
            row,
            '|'.join(' '.join('+' if bit else '-' if bit is None else ' '
                              for bit in image_hash[row:row + row_length])
                     for image_hash in hashes)
        ) for row in range(0, num_bits, row_length)]))

    @classmethod
    def log(cls, msg, level=utils.LOGDEBUG):
        utils.log(msg, name=cls.__name__, level=level)

    def _evaluate_similarity(self, image, filtered_image, hash_size):
        is_match = False
        possible_match = False

        stats = {
            # Similarity to representative end credits hash
            'credits': constants.UNDEFINED,
            # Similarity between detected credits hashes
            'detected': constants.UNDEFINED,
            # Similarity to previous frame hash
            'previous': constants.UNDEFINED,
            # Similarity to hash from other episodes
            'episodes': constants.UNDEFINED
        }

        possible_credits, expanded_image = image_utils.process(
            image,
            queue=[
                [image_utils.entropy_compare, filtered_image, 1.10]
            ],
            save_file='3_expanded'
        )

        image_hash = self._create_hash(image, hash_size)
        filtered_hash = self._create_hash(filtered_image, hash_size)
        expanded_hash = None

        if possible_credits:
            expanded_hash = self._create_hash(expanded_image, hash_size)

            # Calculate similarity between current hash and representative hash
            stats['credits'] = max(self._hash_similarity(
                self.hashes.data.get(self.hash_index['credits_small']),
                image_hash,
                filtered_hash
            ), self._hash_similarity(
                self.hashes.data.get(self.hash_index['credits_large']),
                image_hash,
                filtered_hash
            ), self._hash_similarity(
                self.hashes.data.get(self.hash_index['credits_scroll']),
                image_hash,
                filtered_hash
            ))

            # Estimate of detection relevance
            stats['detected'] = stats['credits'] * self._hash_similarity(
                expanded_hash, image_hash, filtered_hash
            ) / self._hash_similarity(
                filtered_hash, image_hash, expanded_hash
            )

        # Match if current hash matches representative hash or if current hash
        # is blank
        is_match = (
            not any(image_hash)
            or stats['credits'] >= SETTINGS.detect_level
        )
        # Unless debugging, return if match found, otherwise continue checking
        if is_match and not SETTINGS.detector_debug:
            self._hash_match_hit()
            return stats, (image_hash, filtered_hash, expanded_hash)

        # Calculate similarity between current hash and previous hash
        stats['previous'] = self._hash_similarity(
            self.hashes.data.get(self.hash_index['previous']),
            image_hash
        )
        # Possible match if current hash matches previous hash
        possible_match = stats['previous'] >= SETTINGS.detect_level
        # Match if detection estimate indicates result was relevant
        is_match = is_match or (
            possible_match and
            stats['detected'] >= (
                SETTINGS.detect_level -
                (0.004 * stats['previous'] * stats['credits'])
            )
        )
        # Unless debugging, return if match found, otherwise continue checking
        if is_match and not SETTINGS.detector_debug:
            self._hash_match_hit()
            return stats, (image_hash, filtered_hash, expanded_hash)

        old_hashes = self.past_hashes.window(self.hash_index['current'])
        for self.hash_index['episodes'], old_hash in old_hashes.items():
            stats['episodes'] = self._hash_similarity(
                old_hash,
                image_hash
            )
            # Match if current hash matches other episode hashes
            if stats['episodes'] >= SETTINGS.detect_level:
                is_match = True
                break

        # Increment the number of matches
        if is_match:
            self._hash_match_hit()
        # Otherwise increment number of mismatches
        elif not possible_match:
            self._hash_match_miss()

        return stats, (image_hash, filtered_hash, expanded_hash)

    def _hash_match_hit(self):
        with self._lock:
            self.match_counts['hits'] += 1
            self.match_counts['misses'] = 0
            self.match_counts['detected'] = self.match_counts['detected'] or (
                self.match_counts['hits'] >= self.match_number
            )

    def _hash_match_miss(self):
        with self._lock:
            self.match_counts['misses'] += 1
            if self.match_counts['misses'] < self.mismatch_number:
                return
        self._hash_match_reset()

    def _hash_match_reset(self):
        with self._lock:
            self.match_counts['hits'] = 0
            self.match_counts['misses'] = 0
            self.match_counts['detected'] = False

    def _init_hashes(self):
        # Set minimum capture interval to decrease capture rate
        self.capture_interval = 1

        self.hash_index = {
            # Hash indexes are tuples containing the following data:
            # (time_to_end, time_from_start, group_idx)
            # Current hash
            'current': (0, 0, 0),
            # Previous hash
            'previous': None,
            # Representative end credits hashes
            'credits_small': (0, 0, constants.UNDEFINED),
            'credits_large': (0, 1, constants.UNDEFINED),
            'credits_scroll': (0, 2, constants.UNDEFINED),
            # Other episodes hash
            'episodes': None,
            # Detected end credits timestamp from end of file
            'detected_at': None
        }

        # Hash size as (width, height)
        hash_size = [8 * self.get_video_resolution()[2], 8]
        # Round down width to multiple of 2
        hash_size[0] = int(hash_size[0] - hash_size[0] % 2)

        # Hashes for currently playing item
        self.hashes = UpNextHashStore(
            hash_size=hash_size,
            item=self.state.current_item,
            data={
                # Representative hash of centred end credits text on a dark
                # background
                self.hash_index['credits_small']: self._generate_initial_hash(
                    *hash_size,
                    pad_height=(hash_size[1] // 4)
                ),
                self.hash_index['credits_large']: self._generate_initial_hash(
                    *hash_size,
                    pad_height=(hash_size[1] // 8)
                ),
                # Representative hash of scrolling end credits text on a dark
                # background
                self.hash_index['credits_scroll']: self._generate_initial_hash(
                    *hash_size,
                    pad_height=0,
                    pad_width=(hash_size[0] // 4),
                    fuzz_height=0
                ),
            },
        )

        # Hashes from previously played episodes
        self.past_hashes = UpNextHashStore(hash_size=hash_size)
        if SETTINGS.detector_save_path and self.hashes.is_valid():
            self.past_hashes.load(self.hashes.group_name)

        # Number of consecutive frame matches required for a positive detection
        # Set to 5s of captured frames as default
        self.match_number = int(
            SETTINGS.detect_matches / self.capture_interval
        )
        # Number of consecutive frame mismatches required to reset match count
        # Set to 3 frames to account for bad frame capture
        self.mismatch_number = SETTINGS.detect_mismatches
        self._hash_match_reset()

    def _queue_clear(self, queue=None):
        queue = queue or self.queue
        if not queue:
            return

        with queue.mutex:
            queue.queue.clear()
            queue.all_tasks_done.notify_all()
            queue.unfinished_tasks = 0

    def _queue_create(self):
        queue = Queue(maxsize=SETTINGS.detector_threads)
        del self.queue
        self.queue = queue
        return queue

    def _queue_push(self, queue=None):
        queue = queue or self.queue
        try:
            capturer, size = self._queue_pull(queue)
            capturer.capture(*size)
            abort = False
        except TypeError:
            abort = True

        while not (abort or self._sigterm.is_set() or self._sigstop.is_set()):
            loop_start = timeit.default_timer()

            with utils.ContextManager(self, 'player') as (player, error):
                if error is AttributeError:
                    raise error
                error = player.get_speed() != 1
            if error:
                self.log('Stop capture: nothing playing')
                break

            image_data = capturer.getImage()

            # Capture failed or was skipped, retry with less data
            if not image_data or image_data[-1] != 255:
                self.log('Capture failed using {0}kB data limit'.format(
                    SETTINGS.detector_data_limit
                ), utils.LOGWARNING)
                SETTINGS.detector_data_limit = (
                    SETTINGS.detector_data_limit - 8
                ) or 8

                image_data = None
                size = self._get_video_capture_resolution(
                    max_size=SETTINGS.detector_data_limit
                )

                del capturer
                capturer = xbmc.RenderCapture()

            try:
                queue.put((image_data, size), timeout=self.capture_interval)
                capturer.capture(*size)

                loop_time = timeit.default_timer() - loop_start
                if loop_time >= self.capture_interval:
                    raise QueueFull

                abort = utils.wait(self.capture_interval - loop_time)

            except AttributeError:
                self.log('Stop capture: detector stopped')
                break

            except QueueFull:
                self.log('Capture/detection desync', utils.LOGWARNING)
                abort = utils.abort_requested()
                continue

        del capturer
        self._queue_task_done(queue)
        self._queue_clear(queue)

    def _queue_pull(self, queue=None, timeout=None):
        queue = queue or self.queue
        if not queue:
            return None

        return queue.get(timeout=timeout)

    def _queue_task_done(self, queue=None):
        queue = queue or self.queue
        if not queue or not queue.unfinished_tasks:
            return

        queue.task_done()

    @utils.Profiler(enabled=SETTINGS.detector_debug, lazy=True)
    def _worker(self):
        """Detection loop captures Kodi render buffer every 1s to create an
           image hash. Hash is compared to the previous hash to determine
           whether current frame of video is similar to the previous frame.

           Hash is also compared to hashes calculated from previously played
           episodes to detect common sequence of frames (i.e. credits).

           A consecutive number of matching frames must be detected to confirm
           that end credits are playing."""

        queue = self.queue

        while not (self._sigterm.is_set() or self._sigstop.is_set()):
            with utils.ContextManager(self, 'player') as (player, error):
                if error is AttributeError:
                    raise error
                play_time = player.getTime()
                self.hash_index['current'] = (
                    int(player.getTotalTime() - play_time),
                    int(play_time),
                    self.hashes.group_idx
                )
                # Only capture if playing at normal speed
                error = player.get_speed() != 1
            if error:
                self.log('No file is playing')
                break

            try:
                image_data, size = self._queue_pull(queue,
                                                    SETTINGS.detector_threads)
                if not isinstance(image_data, (bytes, bytearray)):
                    raise QueueEmpty
            except TypeError:
                self.log('Queue empty - exiting')
                break
            except QueueEmpty:
                self.log('Queue empty - retry')
                continue

            image, filtered_image = self._create_images(image_data, size)

            # Check if current hash matches with previous hash, typical end
            # credits hash, or other episode hashes
            stats, hashes = self._evaluate_similarity(
                image, filtered_image, self.hashes.hash_size
            )
            image_hash, filtered_hash, expanded_hash = hashes

            if SETTINGS.detector_debug:
                self.log('Match: {0[hits]}/{1}, Miss: {0[misses]}/{2}'.format(
                    self.match_counts, self.match_number, self.mismatch_number
                ))

                self._print_hashes(
                    [filtered_hash,
                     expanded_hash,
                     self.hashes.data.get(self.hash_index['credits_small']),
                     self.hashes.data.get(self.hash_index['credits_large']),
                     self.hashes.data.get(self.hash_index['credits_scroll'])],
                    size=self.hashes.hash_size,
                    prefix=(
                        '{0:.1f}% similar to typical credits, '
                        '{1:.1f}% similarity in detected credits'
                    ).format(stats['credits'], stats['detected'])
                )

                self._print_hashes(
                    [self.hashes.data.get(self.hash_index['previous']),
                     image_hash,
                     self.past_hashes.data.get(self.hash_index['episodes'])],
                    size=self.hashes.hash_size,
                    prefix=(
                        '{0:.1f}% similar to previous hash, '
                        '{1:.1f}% similar to other episodes'
                    ).format(stats['previous'], stats['episodes'])
                )

            # Store current hash for comparison with next video frame
            self.hashes.data[self.hash_index['current']] = image_hash
            self.hash_index['previous'] = self.hash_index['current']

            # Store timestamps if credits are detected
            self.update_timestamp(play_time)

            self._queue_task_done(queue)

        self._queue_task_done(queue)

    def _worker_release(self):
        if not self.workers or not self.queue:
            return

        for idx, worker in enumerate(self.workers):
            if worker.is_alive():
                try:
                    self.queue.put_nowait(None)
                except QueueFull:
                    pass
                worker.join(
                    2 * SETTINGS.detector_threads * self.capture_interval
                )

            if worker.is_alive():
                self.log('Worker {0}({1}) is taking too long to stop'.format(
                    idx, worker.ident
                ), utils.LOGWARNING)

    def is_alive(self):
        return self._running.is_set()

    def cancel(self):
        self.stop()

    def credits_detected(self):
        # Ignore invalidated hash data
        if not self.hashes.is_valid():
            return False

        return self.match_counts['detected']

    @staticmethod
    def get_video_resolution(_cache=[None]):  # pylint: disable=dangerous-default-value
        """Method to detect playing video resolution and aspect ratio"""

        # We don't need to get resolution every time, cache and reuse instead
        if _cache[0] is not None:
            return _cache[0]

        width = xbmc.getInfoLabel('Player.Process(VideoWidth)')
        width = int(width.replace(',', ''))
        height = xbmc.getInfoLabel('Player.Process(VideoHeight)')
        height = int(height.replace(',', ''))
        aspect_ratio = width / height

        resolution = width, height, aspect_ratio
        _cache[0] = resolution
        return resolution

    def reset(self):
        self._hash_match_reset()
        self.hashes.timestamps[self.hashes.group_idx] = None
        self.hash_index['detected_at'] = None

    def start(self, restart=False):
        """Method to run actual detection test loop in a separate thread"""

        if restart or self._running.is_set():
            self.stop()

        # Reset detector data if episode has changed
        if not self.hashes.is_valid(item=self.state.current_item):
            self._init_hashes()

        # If a previously detected timestamp exists then use it
        stored_timestamp = self.past_hashes.timestamps.get(
            self.hashes.group_idx
        )
        if stored_timestamp and not SETTINGS.detector_debug:
            self.log('Stored credits timestamp found')
            self.state.set_detected_popup_time(stored_timestamp)
            utils.event('upnext_credits_detected', internal=True)
            return

        # Otherwise run the detector in a new thread
        with self._lock:
            self.log('Started')
            queue = self._queue_create()
            queue.put_nowait([
                xbmc.RenderCapture(),
                self._get_video_capture_resolution(
                    max_size=SETTINGS.detector_data_limit
                )
            ])
            self.workers = [utils.run_threaded(self._queue_push,
                                               kwargs={'queue': queue})]
            self.workers += [
                utils.run_threaded(self._worker,
                                   delay=start_delay * self.capture_interval)
                for start_delay in range(SETTINGS.detector_threads - 1)
            ]
            self._running.set()

        queue.join()
        self._worker_release()

        self.log('Stopped')
        self._running.clear()
        self._sigstop.clear()
        self._sigterm.clear()

    def stop(self, terminate=False):
        # Set terminate or stop signals if detector is running
        if self._running.is_set():
            if terminate:
                self._sigterm.set()
            else:
                self._sigstop.set()

            self._queue_clear()
            self._worker_release()
            utils.wait(1)

        # Free references/resources
        with self._lock:
            del self.workers
            self.workers = None
            del self.queue
            self.queue = None
            if terminate:
                # Invalidate collected hashes if not needed for later use
                self.hashes.invalidate()
                # Delete reference to instances if not needed for later use
                del self.player
                self.player = None
                del self.state
                self.state = None

    def store_data(self):
        # Only store data for videos that are grouped by season (i.e. same show
        # title, same season number)
        if not self.hashes.is_valid(for_saving=True):
            return

        self.past_hashes.hash_size = self.hashes.hash_size
        self.past_hashes.timestamps.update(self.hashes.timestamps)
        # If credit were detected only store the previous +/- 5s of hashes to
        # reduce false positives when comparing to other episodes
        self.past_hashes.data.update(self.hashes.window(
            self.hash_index['detected_at'], all_episodes=True
        ) if self.match_counts['detected'] else self.hashes.data)

        if SETTINGS.detector_save_path:
            self.past_hashes.save(self.hashes.group_name)

    def update_timestamp(self, play_time):
        # Timestamp already stored or credits not detected
        if self.hash_index['detected_at'] or not self.credits_detected():
            return

        with self._lock:
            self.log('Credits detected')
            self.hash_index['detected_at'] = self.hash_index['current']
            self.hashes.timestamps[self.hashes.group_idx] = play_time
            self.state.set_detected_popup_time(play_time)
            utils.event('upnext_credits_detected', internal=True)


class Solution:
    def rangeAddQueries(self, n: int, queries: List[List[int]]) -> List[List[int]]:
#======== <Solution 1> ========#
        import numpy as np
        mat = np.array([[0] * n for _ in range(n)])
        for row1, col1, row2, col2 in queries:
            mat[row1: row2 + 1, col1: col2 + 1] += 1
        return mat

#======== <Solution 2> ========#
        # Reference: https://leetcode.com/problems/increment-submatrices-by-one/solutions/3052675/python3-sweep-line-range-addition-clean-concise/
        mat = [[0] * n for _ in range(n)]
        for r1, c1, r2, c2 in queries:
            for r in range(r1, r2 + 1):
                mat[r][c1] += 1  # Only add c1 by 1 for each row
                if c2 + 1 < n:  # c2 is not the last column of the matrix
                    mat[r][c2 + 1] -= 1  # Subtract 1 from the next column of c2 to cancel out the addition in later process
        for r in range(n):
            for c in range(1, n):
                mat[r][c] += mat[r][c - 1]  # Prefix sum from previous column
        return mat


import os, sys
os.environ["CUDA_VISIBLE_DEVICES"] = "1" # str(i)
from rtpt import RTPT
import torch
import pandas as pd
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
from torch.utils.data import DataLoader, TensorDataset, SequentialSampler

import util as u
from glob import glob

#####
task_in = 'IMDB'
#task_in = 'Twitter'

# "bertbase", 'bertlarge', "distbase", "distlarge", "robertabase", "robertalarge", "albertbase", "albertlarge"
#model_id_in = "bertbase"
model_id_in = "bertbase"
specs_all = ["original", "N_pro", "N_weat", "N_all", "mix_pro", "mix_weat", "mix_all"]

specs_in = specs_all

def test(model_path,
         spec = None, # if not specified, the test data will correspond to the training data
         batch_size_=32): #task=task_in, model_id=model_id_in, spec=specs_in[0]):
    # df = pd.read_csv("./cola_public/raw/out_of_domain_dev.tsv", delimiter='\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])

    #df = pd.read_pickle(test_data_path)

    # Report the number of sentences.
    # print('Number of test sentences: {:,}\n'.format(df.shape[0]))

    ########## ########## ########## ########## ##########
    if torch.cuda.is_available():
        device = torch.device("cuda")
        print('There are %d GPU(s) available.' % torch.cuda.device_count())
        print('We will use the GPU:', torch.cuda.get_device_name(0))
    else:
        print('No GPU available, using the CPU instead.')
        device = torch.device("cpu")
    ########## ########## ########## ########## ##########

    task = u.identify_task(model_path)
    model_id = u.identify_model_id(model_path)
    if not spec:
        spec = u.identify_spec(model_path)

    sys.stdout = open(model_path + '/evaluation_{}_log.txt'.format(spec), 'a')

    # Create sentence and label lists
    sentences, labels = u.import_data('test', task, spec)
    print('labels: ', set(labels), labels.tolist().count(0), labels.tolist().count(1))
    print('Number of test sentences: {:,}\n'.format(len(sentences)))
    print('Number of test labels: {:,}\n'.format(len(labels)))

    # Tokenize all of the sentences and map the tokens to thier word IDs.
    input_ids = []
    attention_masks = []

    #print(model_id)

    tokenizer, model = u.load_hf(model_id, path_to_model=model_path) # Todo: load tokenizer from device, not from HF
    # For every sentence...
    for sent in sentences:
        # same as in training
        encoded_dict = tokenizer.encode_plus(
                            sent,                      # Sentence to encode.
                            add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                            max_length = 64,           # Pad & truncate all sentences.
                            pad_to_max_length = True,
                            return_attention_mask = True,   # Construct attn. masks.
                            return_tensors = 'pt',     # Return pytorch tensors.
                       )

        input_ids.append(encoded_dict['input_ids'])
        attention_masks.append(encoded_dict['attention_mask'])

    # Convert the lists into tensors.
    input_ids = torch.cat(input_ids, dim=0).to(device)
    attention_masks = torch.cat(attention_masks, dim=0).to(device)
    labels = torch.tensor(labels).to(device)

    # Set the batch size.
    batch_size = batch_size_

    # Create the DataLoader.
    prediction_data = TensorDataset(input_ids, attention_masks, labels)
    prediction_sampler = SequentialSampler(prediction_data)
    prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)

    print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))

    #model = model_class.from_pretrained()
    model.eval()# Put model in evaluation mode
    model.to(device)

    # Tracking variables
    predictions , true_labels = [], []

    save_dict = {"batch": [], "accuracy": [], "recall": [], "precision": [], "f1":[]}

    # Predict
    i = 0
    for batch in prediction_dataloader:
        # Add batch to GPU
        batch = tuple(t.to(device) for t in batch)
        # Unpack the inputs from our dataloader
        b_input_ids, b_input_mask, b_labels = batch

        b_input_ids = b_input_ids.to(device)
        b_input_mask = b_input_mask.to(device)
        b_labels = b_labels.to(device)

        # Telling the model not to compute or store gradients, saving memory and
        # speeding up prediction
        with torch.no_grad():
            # Forward pass, calculate logit predictions
            outputs = model(b_input_ids, #token_type_ids=None,
                            attention_mask=b_input_mask)

        logits = outputs[0]

        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        # Store predictions and true labels
        predictions.extend(logits)
        true_labels.extend(label_ids)

        #logits_ = [0 if x>y else 1 for (x,y) in logits]
        #label_ids_ = label_ids.tolist()

        #batch_acc = accuracy_score(y_true=label_ids_, y_pred=logits_)
        #batch_rec = recall_score(y_true=label_ids_, y_pred=logits_)
        #batch_pre = precision_score(y_true=label_ids_, y_pred=logits_)
        #batch_f1 = f1_score(y_true=label_ids_, y_pred=logits_)

        #save_dict["batch"].append(i)
        #save_dict["accuracy"].append(batch_acc)
        #save_dict["recall"].append(batch_rec)
        #save_dict["precision"].append(batch_pre)
        #save_dict["f1"].append(batch_f1)

        i+=1
        #print("batch result: accuracy = {}; recall = {}; precision = {}; f1 = {}".format(batch_acc,batch_rec, batch_pre, batch_f1 ))

    # print(label.sum(), len(df.label), (df.label.sum() / len(df.label) * 100.0))
    predictions = [0 if x>y else 1 for (x,y) in predictions]
    label_ids = label_ids.tolist()
    acc = accuracy_score(y_true=true_labels, y_pred=predictions)
    rec = recall_score(y_true=true_labels, y_pred=predictions)
    pre = precision_score(y_true=true_labels, y_pred=predictions)
    f1 = f1_score(y_true=true_labels, y_pred=predictions)
    save_dict["batch"].append('final')
    save_dict["accuracy"].append(acc)
    save_dict["recall"].append(rec)
    save_dict["precision"].append(pre)
    save_dict["f1"].append(f1)

    res_df = pd.DataFrame(save_dict)
    res_df.to_pickle(model_path + '/evaluation_'+spec)

    print('    DONE.')
    return acc, rec, pre, f1

all_model_paths = glob("res_models/models/*") # cloud also be restricted by hand
all_model_paths = [x for x in all_model_paths if 'IMDB' in x ] # only IMDB for now, because twitter is still in training

all_model_paths = [
    "res_models/models/IMDB_distbase_N_pro",
    "res_models/models/IMDB_albertbase_N_pro",
    "res_models/models/IMDB_bertlarge_mix_weat",
    "res_models/models/IMDB_bertlarge_mix_pro",
    "res_models/models/IMDB_albertlarge_mix_pro",
    "res_models/models/IMDB_bertlarge_N_pro",
    "res_models/models/IMDB_bertlarge_N_weat",
    "res_models/models/IMDB_robertabase_N_all",
    "res_models/models/IMDB_robertalarge_N_pro",
    "res_models/models/IMDB_albertbase_N_all",
    "res_models/models/IMDB_distbase_N_all",
    "res_models/models/IMDB_distbase_mix_pro",
    "res_models/models/IMDB_bertlarge_N_all",
    "res_models/models/IMDB_bertbase_original",
    "res_models/models/IMDB_albertbase_original",
    "res_models/models/IMDB_bertbase_N_all",
    "res_models/models/IMDB_albertlarge_N_all",
    "res_models/models/IMDB_albertlarge_N_pro",
    "res_models/models/IMDB_albertlarge_mix_weat",
    "res_models/models/IMDB_bertbase_N_weat",
    "res_models/models/IMDB_albertlarge_original",
    "res_models/models/IMDB_robertalarge_mix_all",
    "res_models/models/IMDB_albertbase_mix_weat",
    "res_models/models/IMDB_robertalarge_original",
    "res_models/models/IMDB_robertalarge_N_weat",
    "res_models/models/IMDB_bertbase_mix_weat",
    "res_models/models/IMDB_distbase_N_weat",
    "res_models/models/IMDB_robertalarge_mix_pro",
    "res_models/models/IMDB_robertabase_N_weat",
    "res_models/models/IMDB_albertbase_mix_pro",
    "res_models/models/IMDB_distbase_mix_all",
    "res_models/models/IMDB_robertalarge_N_all",
    "res_models/models/IMDB_bertlarge_mix_all",
    "res_models/models/IMDB_bertbase_mix_pro",
    "res_models/models/IMDB_distbase_mix_weat",
    "res_models/models/IMDB_robertabase_N_pro",
    "res_models/models/IMDB_robertabase_mix_weat",
    "res_models/models/IMDB_distbase_original",
    "res_models/models/IMDB_albertbase_N_weat",
    "res_models/models/IMDB_albertlarge_mix_all",
    "res_models/models/IMDB_robertabase_mix_pro",
    "res_models/models/IMDB_robertalarge_mix_weat",
    "res_models/models/IMDB_robertabase_original",
    "res_models/models/IMDB_bertlarge_original",
    "res_models/models/IMDB_albertbase_mix_all",
    "res_models/models/IMDB_bertbase_mix_all",
    "res_models/models/IMDB_albertlarge_N_weat",
    "res_models/models/IMDB_robertabase_mix_all",
    "res_models/models/IMDB_bertbase_N_pro"
    ]

all_model_paths_ = [
    ['res_models/models_BS/IMDB_bertbase_N_all_BS08'], #/e20do5_epoch4'] ,
    ['res_models/models_BS/IMDB_bertbase_N_all_BS16'], #/e20do5_epoch11'] ,
    ['res_models/models_BS/IMDB_bertbase_N_all_BS32'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_bertbase_N_pro_BS08'], #/e20do5_epoch4'] ,
    ['res_models/models_BS/IMDB_bertbase_N_pro_BS16'], #/e20do5_epoch11'] ,
    ['res_models/models_BS/IMDB_bertbase_N_pro_BS32'], #/e20do5_epoch5'] ,
    ['res_models/models_BS/IMDB_bertbase_N_weat_BS08'], #/e20do5_epoch6'] ,
    ['res_models/models_BS/IMDB_bertbase_N_weat_BS16'], #/e20do5_epoch10'] ,
    ['res_models/models_BS/IMDB_bertbase_N_weat_BS32'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_all_BS08'], #/e20do5_epoch13'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_all_BS16'], #/e20do5_epoch18'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_all_BS32'], #/e20do5_epoch12'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_pro_BS08'], #/e20do5_epoch16'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_pro_BS16'], #/e20do5_epoch18'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_pro_BS32'], #/e20do5_epoch5'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_weat_BS08'], #/e20do5_epoch17'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_weat_BS16'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_bertbase_mix_weat_BS32'], #/e20do5_epoch11'] ,
    ['res_models/models_BS/IMDB_bertbase_original_BS08'], #/e20do5_epoch4'] ,
    ['res_models/models_BS/IMDB_bertbase_original_BS16'], #/e20do5_epoch11'] ,
    ['res_models/models_BS/IMDB_bertbase_original_BS32'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_all_BS08'], #/e20do5_epoch17'] ,
    ['res_models/models_BS/IMDB_distbase_mix_all_BS16'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_all_BS32'], #/e20do5_epoch14'] ,
    ['res_models/models_BS/IMDB_distbase_mix_pro_BS08'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_pro_BS16'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_pro_BS32'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_weat_BS08'], #/e20do5_epoch15'] ,
    ['res_models/models_BS/IMDB_distbase_mix_weat_BS16'], #/e20do5_epoch19'] ,
    ['res_models/models_BS/IMDB_distbase_mix_weat_BS32'], #/e20do5_epoch19']
]

rtpt = RTPT(name_initials='SJ', experiment_name='evaluation', max_iterations=len(all_model_paths))
rtpt.start()

for path in all_model_paths:
    model_path=path
    for file in glob(path+ "/*"):
        if 'epoch' in file:
            model_path = file
    print(model_path)
    res_ = test(model_path)
    res_original = test(model_path, spec='original')
    rtpt.step()

# TODO: write function that collects all evaluations and creates a single data frame from all finals - no pytorch needed, only pandas work.


from drawable import Drawable
from positionable import Positionable

class Base(Drawable, Positionable):
  SIZE = 40
  COLOR = "blue"

  def __init__(self, x, y):
    super(Drawable, self).__init__(x, y, self.SIZE)

  def draw(self, canvas):
    canvas.create_rectangle(self.x - self.size / 2,
                            self.y - self.size / 2,
                            self.x + self.size / 2,
                            self.y + self.size / 2,
                            fill=self.COLOR)


# -*- coding: utf-8 -*-

__author__ = 'mayns'

# ---------- SCIENCE FIELDS ---------- #

V_MATH = u'Математика, информатика и механика'
V_PHYSICS = u'Физика и астрономия'
V_CHEMISTRY = u'Химия'
V_BIOLOGY = u'Биология и медицинская наука'
V_EARTH = u'Науки о Земле'
V_ENGINEERING = u'Инженерные науки'
V_IT = u'Информационные технологии и вычислительные системы'

K_MATH = u'math'
K_PHYSICS = u'physics'
K_CHEMISTRY = u'chemistry'
K_BIOLOGY = u'biology'
K_EARTH = u'earth'
K_ENGINEERING = u'engineering'
K_IT = u'it'

SCIENCE_FIELDS_MAP = {
    K_MATH: V_MATH,
    K_PHYSICS: V_PHYSICS,
    K_CHEMISTRY: V_CHEMISTRY,
    K_BIOLOGY: V_BIOLOGY,
    K_EARTH: V_EARTH,
    K_ENGINEERING: V_ENGINEERING,
    K_IT: V_IT,
}

# ---------- CONTACT TYPES ----------- #

PHONE = u'phone'
SKYPE = u'skype'
EMAIL = u'email'

CONTACT_TYPES = [PHONE, SKYPE, EMAIL]

# ------------ IMAGE SIZES ------------ #

AVATAR_SIZES = [60, 100, 250]

IMG_S = 60
IMG_M = 100
IMG_L = 250

GET_IMG = lambda url, size: u'{url}{size}.png'.format(url=url, size=size) if not u'gravatar' in url \
    else u'{url}?size={size}'.format(url=url, size=size)

IMAGE_URL_MENU = lambda img_url: u'{}60.png'.format(img_url)
IMAGE_URL_LIST = lambda img_url: u'{}100.png'.format(img_url)
IMAGE_URL_PROFILE = lambda img_url: u'{}250.png'.format(img_url)

AVATAR_PATH = lambda sc_id: u'{sc_id}/a'.format(sc_id=str(sc_id))
MEDIA_USR_PATH = lambda sc_id: str(sc_id)

DATETIME_FORMAT = dict(
    date=dict(
        DB=u'%Y-%m-%d',
        HUMAN=u'%d.%m.%Y'
    ),
    timestamp=dict(
        DB=u'%Y-%m-%d %H:%M:%S',
        HUMAN=u'%d.%m.%Y %H:%M:%S'
    )
)

TABLE_ROLES = u'roles'
TABLE_VACANCIES = u'vacancies'
TABLE_PARTICIPANTS = u'participants'
TABLE_RESPONSES = u'responses'

ROLE_ADMIN = 1
ROLE_USER = 2

STATUS_ACCEPTED = u'accepted'
STATUS_DECLINED = u'declined'
STATUS_WAITING = u'waiting'
STATUS_DELETED = u'deleted'
STATUS_ACTIVE = u'active'
STATUS_ARCHIVED = u'archived'


#################################
# EVON API Client
#################################

import base64
import json
import logging
import os
import subprocess

from dotenv import dotenv_values
import requests

from evon import log

logger = log.get_evon_logger()
EVON_DEBUG = os.environ.get('EVON_DEBUG', '').upper() == "TRUE"
if EVON_DEBUG:
    logger.setLevel(logging.DEBUG)
# API_URL = os.environ.get("EVON_API_URL")
REQUESTS_TIMEOUT = 30
evon_env = dotenv_values(os.path.join(os.path.dirname(__file__), ".evon_env"))
STANDALONE_MODE = evon_env["STANDALONE"] == "True"
STANDALONE_HOOK_PATH = evon_env["STANDALONE_HOOK_PATH"]

class BytesEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, bytes):
            return obj.decode("utf-8")
        return json.JSONEncoder.default(self, obj)

def generate_headers(api_key):
    response = requests.get(
        "http://169.254.169.254/latest/dynamic/instance-identity/document",
        timeout=REQUESTS_TIMEOUT
    )
    iid = base64.b64encode(response.text.encode("utf-8"))
    response = requests.get(
        "http://169.254.169.254/latest/dynamic/instance-identity/signature",
        timeout=REQUESTS_TIMEOUT
    )
    iid_signature = response.text.replace("\n", "")
    headers = {
        "Content-Type": "application/json",
        "X-API-Key": api_key,
        "document": iid,
        "signature": iid_signature
    }
    logger.debug(f"headers are: {headers}")
    return headers

def get_pub_ipv4():
    response = requests.get(
        "http://169.254.169.254/latest/meta-data/public-ipv4",
        timeout=REQUESTS_TIMEOUT
    )
    return response.text

def do_request(url, requests_method, headers, json_payload=None, params={}):
    if STANDALONE_MODE:
        logger.info(f"standalone mode enabled, calling standalone hook at path: {STANDALONE_HOOK_PATH}")
        env = {
            **os.environ,
            "EVON_HOOK_REQUEST_URL": url,
            "EVON_HOOK_REQUEST_METHOD": requests_method.__name__,
            "EVON_HOOK_HEADERS": json.dumps(headers, cls=BytesEncoder),
            "EVON_HOOK_BODY": json_payload or "",
            "EVON_HOOK_PARAMS": json.dumps(params),
        }
        p = subprocess.Popen(STANDALONE_HOOK_PATH, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, close_fds=True, encoding="utf-8")
        rc = p.wait()
        stdout = p.stdout.read() or ""
        stderr = p.stderr.read() or ""
        logger.info(f"results after executing standalone hook: rc: {rc}, stdout: {stdout}, stderr: {stderr}")
        return stdout
    else:
        request_kwargs = {
            "headers": headers,
        }
        if json_payload:
            request_kwargs["data"] = json_payload.encode("utf-8")
        if params:
            request_kwargs["params"] = params
        response = None
        try:
            response = requests_method(url, **request_kwargs, timeout=REQUESTS_TIMEOUT)
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            logger.error(f"{requests_method.__name__.upper()} request failed: '{e}' ")
        return response.text

def get_records(api_url, api_key):
    url = f"{api_url}/zone/records"
    response = do_request(
        url,
        requests.get,
        headers=generate_headers(api_key)
    )
    records = json.dumps(json.loads(response), indent=2)
    return records

def set_records(api_url, api_key, json_payload, usage_stats=False):
    url = f"{api_url}/zone/records"
    if usage_stats:
        url += "?usage_stats=true"
    response = do_request(
        url,
        requests.put,
        headers=generate_headers(api_key),
        json_payload=json_payload
    )
    return response

def register(api_url, api_key, json_payload):
    url = f"{api_url}/zone/register"
    response = do_request(
        url,
        requests.post,
        headers=generate_headers(api_key),
        json_payload=json_payload
    )
    return response

def deregister(api_url, api_key, json_payload):
    url = f"{api_url}/zone/deregister"
    response = do_request(
        url,
        requests.delete,
        headers=generate_headers(api_key),
        json_payload=json_payload
    )
    return response

def get_updates(api_url, api_key, version, selfhosted=False):
    url = f"{api_url}/zone/update/{version}"
    response = do_request(
        url,
        requests.get,
        headers=generate_headers(api_key),
        params={"selfhosted": selfhosted}
    )
    return response

def get_meters(api_url, api_key):
    url = f"{api_url}/zone/meters"
    response = do_request(
        url,
        requests.get,
        headers=generate_headers(api_key)
    )
    return response

def get_usage_limits(api_url, api_key):
    url = f"{api_url}/zone/meters?usage_limits=true"
    response = do_request(
        url,
        requests.get,
        headers=generate_headers(api_key)
    )
    return response


import argparse
import time

import matplotlib.pyplot as plt
import torch
import torch.nn.functional as F
from matplotlib import rc

from gsoftmax.modules import Gspace2d

rc('font', **{'family': 'sans-serif'})
rc('text', usetex=True)
rc('text.latex', preamble=r'\usepackage{amsmath}')

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--device', type=str, help='device')

args = parser.parse_args()

def make_random_alpha(batch_size, h, w):
    alpha = torch.empty((batch_size, h, w)).uniform_(0, 1.)
    supp = torch.empty((batch_size, h, w)).uniform_(0, 1.) > 0.9
    alpha[~supp] = 0
    alpha /= alpha.view(batch_size, -1).sum(dim=1)[:, None, None]
    return alpha

def make_unidim_alpha(batch_size, h, w):
    alpha = torch.zeros((batch_size, h, w))
    alpha[:, h // 5, w // 5: (4 * w) // 5] = 1
    alpha[:, (4 * h) // 5, w // 5: (4 * w) // 5] = 1
    alpha[:, w // 5: (4 * w) // 5, w // 5] = 1
    alpha[:, w // 5: (4 * w) // 5, (4 * w) // 5] = 1
    alpha /= alpha.view(batch_size, -1).sum(dim=1)[:, None, None]
    return alpha

device = args.device
h, w = 32, 32
batch_size = 1
alpha = make_unidim_alpha(batch_size, h, w)
alpha += make_random_alpha(batch_size, h, w) / 5
alpha /= alpha.view(batch_size, -1).sum(dim=1)[:, None, None]
alpha = alpha[:, None]

alpha = alpha.to(device)

gspace = Gspace2d(h, w, sigma=3, max_iter=1000, tol=0,
                  method='lbfgs', logspace=True, verbose=20, )
metric_space = gspace.to(device)
alpha = alpha.to(device)

alpha = alpha.view(batch_size, -1)
v, f = metric_space.entropy_and_potential(alpha)
for i in range(1):
    t0 = time.perf_counter()
    v, proj = gspace.lse_and_softmax(f)
    elapsed = time.perf_counter() - t0
    print(f'Time: {elapsed:.2f}')

sm = F.softmax(f.view(batch_size, -1), dim=1).view(batch_size, 1, h, w)

alpha = alpha.view(batch_size, 1, h, w)
proj = proj.view(batch_size, 1, h, w)
f = f.view(batch_size, 1, h, w)

alpha = alpha.cpu().numpy()
proj = proj.cpu().numpy()
f = f.cpu().numpy()

fig, axes = plt.subplots(1, 4, figsize=(5, 1.2))
fig.subplots_adjust(left=0.0, right=1, top=1, bottom=0.2, wspace=0.02)
axes = axes.ravel()
axes[0].imshow(alpha[0, 0])

axes[0].annotate(r'Distribution $\alpha$', xy=(0.5, -0.1), ha='center',
                 va='top',
                 xycoords='axes fraction'
                 )
axes[1].imshow(-f[0, 0])
axes[1].annotate('Distance field \n' + r'$f = \nabla \Omega(\alpha)$',
                 va='top', xy=(0.5, -0.1), ha='center',
                 xycoords='axes fraction'
                 )
axes[2].imshow(proj[0, 0])
axes[2].annotate('Link function \n' + r'$\nabla \Omega^\star(f)$',
                 xy=(0.5, -0.1), ha='center',
                 va='top',
                 xycoords='axes fraction'
                 )
axes[3].imshow(sm[0, 0])
axes[3].annotate('Naive \n' + r'$\text{softmax}(f)$', xy=(0.5, -0.1),
                 ha='center',
                 va='top',
                 xycoords='axes fraction')
for ax in axes:
    ax.axis('off')
plt.savefig('link.pdf')


# Import libraries
import numpy as np
import matplotlib.pyplot as plt
import logging
from sklearn import svm, tree, ensemble, metrics
from sklearn.linear_model import LogisticRegression
logger = logging.getLogger(__name__)

from utils import top_precision, top_idx, top_accuracy

# Function desc: build_model function trains the model on the train data for different algorithms and hyperparameter combinations
# Input: train dataset features dataframe, label for train dataset, name of algorithm, hyperparameters and top 10% as default
# Return: Trained model
def build_model(features_train, label_train, model_name, param, topP=0.1):
    target = np.array(label_train)
    features = np.array(features_train)
    # Calculate the default classifier train accuracy
    logger.info(f'\nUsing {model_name} model')
    logger.info(f'Number of students: {len(target)}, Not_graduated: {target.sum()}')
    logger.info(f'Constant classifier train accuracy: {max(target.sum(), len(target)-target.sum()) / len(target)}')
    # Train the model
    if model_name =='Logistic':
        clf = LogisticRegression(solver = param['solver'], C = param['C']).fit(features, target)
    elif model_name == 'SVM':
        clf = svm.SVC(C = param['C'], kernel = param['kernel'], probability = True).fit(features, target)
    elif model_name == 'DecisionTree':
        clf = tree.DecisionTreeClassifier(criterion = param['criterion'], max_depth = param['max_depth']).fit(features, target)
    elif model_name == 'RandomForest':
        clf = ensemble.RandomForestClassifier(n_estimators = param['n_estimators'], max_depth = param['max_depth'],
                                              min_samples_split = param['min_samples_split'],
                                              max_features = param['max_features'], n_jobs = 20,
                                              random_state = 1).fit(features, target)
    else:
        logger.error(f"Model name \'{model_name}\' unclear ")
        raise Exception(f"Model name \'{model_name}\' unclear ")

    logger.info(f'Train Accuracy: {clf.score(features, target)}')
    return clf

# Function desc: prediction function precits the probability of student not graduating
# Input: List of features from test dataset, the trained model
# Return: Array of prediction probabilities for not graduating
def prediction(features_test, model):
    features = np.array(features_test)
    pred_proba = model.predict_proba(features_test)[:,1]

    return pred_proba

# Function desc: evaluate_model function calculates precision score and plots precision recall plot
# Input: 1. List of features in test data, 2. Label of test data, 3. Trained model, 4. Name of the algorithm, 5. Baseline score,
#        6. Hyperparameters for the model, 7. Top 10% as default, 3. Optional column names
# Return: Top 10% precision score
def evaluate_model(features_test, label_test, model, model_name, baseline, hyperparameters, topP=0.1, columns=None):
    pred_proba = prediction(features_test, model)

    # Calculate traditional accuracy
    cut_off = 0.5
    pred_labels = [1 if i > cut_off else 0 for i in pred_proba]
    accuracy = metrics.accuracy_score(label_test,pred_labels)
    logger.info(f'\nClassic Test Accuracy: {accuracy}')
    logger.info(f'\nHyperparameters = {hyperparameters}')

    #compute precision amongst riskiest students
    topK = int(topP * len(label_test))
    accuracy = top_precision(label_test, pred_proba, topP=topP)
    logger.info(f'{topP} precision: {accuracy}\tBaseline: {baseline}\tBest possible precision: {min(np.sum(label_test), topK) / topK}')

    #plot precision recall curve
    percents = [0.01 * i for i in range(101)]
    ps = [top_precision(label_test, pred_proba, topP = p) for p in percents]
    rs = [top_accuracy(label_test, pred_proba, topP = p) for p in percents]
    Ks = [int(len(label_test) * p) for p in percents]
    plt.plot(percents, ps, label="Precision")
    plt.plot(percents, rs, label="Recall")
    plt.xlabel("Top Percentile")
    plt.ylabel("Precision/Recall")
    plt.title("Precision vs. Top K for {model_name} Model")
    plt.legend()
    #plt.savefig(f"../viz/precision-recall-topk-{model_name}-{hyperparameters}.png")
    plt.clf()

    if columns is not None:
        cross_tab(columns, features_test, pred_proba, topP=topP)

    return accuracy

# Function desc: cross_tab function calculates the cross tab scores for features with highest importance scores
# Input: 1. List of columns, 2. List of features for test data, 3. Prediction probabilities, 4. Default 10%, 5. Number of crosstabs
# Return: None
def cross_tab(columns, features_test, pred_proba, topP=0.1, num_tabs=10):
    features_test = np.array(features_test)
    riskiest_idx = top_idx(pred_proba, topP=topP)
    riskiest_features = features_test[riskiest_idx]
    bottom_features = features_test[[i for i in range(len(pred_proba)) if i not in riskiest_idx]]

    riskiest_features = np.mean(riskiest_features, axis=0)
    bottom_features = np.mean(bottom_features, axis=0)

    diff = np.abs(riskiest_features - bottom_features)
    #diff = np.abs(1 - (riskiest_features / bottom_features))
    #diff = np.abs((riskiest_features - bottom_features) / np.std(features_test, axis=0))
    diff_idx = np.flip(np.argsort(diff))

    logger.info("\nCross Tab Results")
    for i in range(num_tabs):
        feature_num = diff_idx[i]
        logger.info(f'Feature = {columns[feature_num]}:\tTopK mean = {riskiest_features[feature_num]}\tBottom mean = {bottom_features[feature_num]}')

# Function desc: baseline function creates precision recall curve for baseline rates calculated using 9th grade GPA
# Input: Test dataframe and year list (cohort) for which the baseline precision recall is calculated
# Return: Precision score
def baseline(df, years):
    # takes the validation dataset and plots a baseline plot using grade 9 gpa

    baseline_df = df[['9_gpa', 'not_graduated']].sort_values('9_gpa').reset_index().drop('index', axis = 1)
    total_drop_outs = sum(baseline_df.not_graduated)
    rnge = [0.01 * i for i in range(101)]

    precision = []
    recall = []
    # Calculate precision and recall for baseline data
    for k in rnge:
        first_k = int(round(k*len(baseline_df), 0))
        precision.append(baseline_df.iloc[:first_k,1].sum(axis = 0)/first_k)
        recall.append(baseline_df.iloc[:first_k,1].sum(axis = 0)/total_drop_outs)

    # Plot the precision recall curve for baseline data
    plt.plot(rnge, precision, label = 'Precision')
    plt.plot(rnge, recall, label = 'Recall')
    plt.title('Baseline rates using only 9th grade gpa')
    plt.xticks(np.arange(0.0, 1.01, 0.2))
    plt.yticks(np.arange(0.0, 1.01, 0.2))
    plt.xlabel('Top Percentile')
    plt.ylabel('Precision/Recall')
    plt.legend()
    plt.savefig(f'../viz/baseline-9gpa-precision-recall-{str(years[1])}-{str(years[0])}.png')
    plt.clf()

    return precision[11]


import requests
import json

#returns a dict of all champion information required.
def getItemInfo():
    url = "http://ddragon.leagueoflegends.com/cdn/10.16.1/data/en_US/item.json"

    #get item info
    response = requests.get(url)
    data = json.loads(response.content)

    requiredAttributes=["name","plaintext"]

    retval={}

    for itemId in data['data']:
        retval[itemId]={}
        for attribute in requiredAttributes:
            retval[itemId][attribute]=data['data'][itemId][attribute]

    return retval

def itemBuildsInto():
    url = "http://ddragon.leagueoflegends.com/cdn/10.16.1/data/en_US/item.json"

    #get item info
    response = requests.get(url)
    data = json.loads(response.content)

    retval={}

    for itemId in data['data']:
        if 'into' in data['data'][itemId]:
            retval[itemId]=data['data'][itemId]['into']

    return retval

f=open('items_builds_into.txt','w')
f.write(json.dumps(itemBuildsInto()))


from argparse import ArgumentParser
from collections import defaultdict

def main():
    p = ArgumentParser("Restore sentences from word ids.")
    p.add_argument(
            '--input',
            type=str, metavar="FILE", required=True, help="source word ids file")
    p.add_argument(
            '--output',
            type=str, metavar="FILE", required=True, help="converted sentence file")
    p.add_argument(
            '--vocab',
            type=str, metavar="FILE", required=True, help="vocab file")
    args = p.parse_args()

    with open(args.vocab, 'r') as f:
        vocab = defaultdict(lambda: "<UNK>")
        for line in f:
            word_id, word = line.rstrip("\n").split("\t")
            vocab[word_id] = word

    with open(args.input, 'r') as input_file, open(args.output, 'w') as output_file:
        for input_line in input_file:
            words = [vocab[x] for x in input_line.rstrip("\n").split()]
            print(" ".join(words), file=output_file)

if __name__ == "__main__":
    main()


import socket
import pickle
import argparse
import time
import os
from functools import reduce
import numpy as np
from scipy.special import logit
import matplotlib as mpl
# mpl.use('Agg')
# import matplotlib.pyplot as plt
from tqdm import tqdm
import mxnet as mx
# from sklearn.neighbors import NearestNeighbors
# from IPython import embed
import collections

def gpu_helper(gpu):
    if gpu >= 0 and gpu_exists(gpu):
        model_ctx = mx.gpu(gpu)
    else:
        model_ctx = mx.cpu()
    return model_ctx

def gpu_exists(gpu):
    try:
        mx.nd.zeros((1,), ctx=mx.gpu(gpu))
    except:
        return False
    return True

def reverse_dict(d):
    return {v:k for k,v in d.items()}

def to_numpy(X):
    x_npy = []
    for x in X:
        if isinstance(x,list):
            x_npy += [to_numpy(x)]
        else:
            x_npy += [x.asnumpy()]
    return x_npy

def stack_numpy(X,xnew):
    for i in range(len(X)):
        if isinstance(xnew[i],list):
            X[i] = stack_numpy(X[i], xnew[i])
        else:
            X[i] = np.vstack([X[i], xnew[i]])
    return X

def get_topic_words_decoder_weights(D, data, ctx, k=10, decoder_weights=False):
    if decoder_weights:
        params = D.collect_params()
        params = params['decoder0_dense0_weight'].data().transpose()
    else:
        y = D.y_as_topics()
        params = D(y.copyto(ctx))
    top_word_ids = mx.nd.argsort(params, axis=1, is_ascend=False)[:,:k].asnumpy()
    if hasattr(data, 'id_to_word'):
        top_word_strings = [[data.id_to_word[int(w)] for w in topic] for topic in top_word_ids]
    else:
        top_word_strings = [[data.maps['dim2vocab'][int(w)] for w in topic] for topic in top_word_ids]

    return top_word_strings

def get_topic_words(D, data, ctx, k=10):
    y, z = D.yz_as_topics()
    if z is not None:
        params = D(y.copyto(ctx), z.copyto(ctx))
    else:
        params = D(y.copyto(ctx), None)
    top_word_ids = mx.nd.argsort(params, axis=1, is_ascend=False)[:,:k].asnumpy()
    if hasattr(data, 'id_to_word'):
        top_word_strings = [[data.id_to_word[int(w)] for w in topic] for topic in top_word_ids]
    else:
        top_word_strings = [[data.maps['dim2vocab'][int(w)] for w in topic] for topic in top_word_ids]

    return top_word_strings

def calc_topic_uniqueness(top_words_idx_all_topics):
    """
    This function calculates topic uniqueness scores for a given list of topics.
    For each topic, the uniqueness is calculated as:  (\sum_{i=1}^n 1/cnt(i)) / n,
    where n is the number of top words in the topic and cnt(i) is the counter for the number of times the word
    appears in the top words of all the topics.
    :param top_words_idx_all_topics: a list, each element is a list of top word indices for a topic
    :return: a dict, key is topic_id (starting from 0), value is topic_uniquness score
    """
    n_topics = len(top_words_idx_all_topics)

    # build word_cnt_dict: number of times the word appears in top words
    word_cnt_dict = collections.Counter()
    for i in range(n_topics):
        word_cnt_dict.update(top_words_idx_all_topics[i])

    uniqueness_dict = dict()
    for i in range(n_topics):
        cnt_inv_sum = 0.0
        for ind in top_words_idx_all_topics[i]:
            cnt_inv_sum += 1.0 / word_cnt_dict[ind]
        uniqueness_dict[i] = cnt_inv_sum / len(top_words_idx_all_topics[i])

    return uniqueness_dict

def request_pmi(topic_dict=None, filename='', port=1234):
    try:
        # create a socket object
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        # get local machine name
        host = socket.gethostname()
        # host = socket.gethostbyname('localhost')

        # connection to hostname on the port.
        s.connect((host, port))

        if filename != '':
            s.sendall(pickle.dumps(filename), )
        else:
            s.send(pickle.dumps(topic_dict), )

        data = []
        while True:
            packet = s.recv(4096)
            # time.sleep(1.0)
            # print('looking at packet # {0}'.format(len(data)))
            # print(packet)
            # print(type(packet))
            wait = len(packet)
            if not packet:
                # embed()
                break
            data.append(packet)
            # print('received packet # {0}'.format(len(data)))
            # time.sleep(1.0)
        res_dict = pickle.loads(b"".join(data))

        s.close()
        pmi_dict = res_dict['pmi_dict']
        npmi_dict = res_dict['npmi_dict']
    except:
        # print('Failed to run NPMI calc, NPMI and PMI set to 0.0')
        pmi_dict = dict()
        npmi_dict = dict()
        for k in topic_dict:
            pmi_dict[k] = 0
            npmi_dict[k] = 0
        # embed()

    return pmi_dict, npmi_dict

def print_topics(topic_json, npmi_dict, topic_uniqs, data, print_topic_names=False):
    for k,v in topic_json.items():
        prefix_msg = '[ '
        if hasattr(data, 'maps') and print_topic_names:
            prefix_msg += data.maps['dim2topic'][k]
        else:
            prefix_msg += str(k)
        if hasattr(data, 'selected_topics') and print_topic_names:
            if data.maps['dim2topic'][k] in data.selected_topics:
                prefix_msg += '*'
        prefix_msg += ' - '
        prefix_msg += '{:.5g}'.format(topic_uniqs[k])
        prefix_msg += ' - '
        prefix_msg += '{:.5g}'.format(npmi_dict[k])
        prefix_msg += ']: '
        print(prefix_msg, v)

def print_topic_with_scores(topic_json, **kwargs):
    """

    :param topic_json:
    :param kwargs: dict_name: content_dict; special argument sortby='xxx' will enable descending sort in printed result
    :return:
    """
    topic_keys = sorted(list(topic_json.keys()))

    sortby = kwargs.pop('sortby', None)
    if sortby is None:
        sortby = kwargs.pop('sort_by', None)

    if sortby in kwargs.keys():
        topic_keys = sorted(kwargs[sortby], key=kwargs[sortby].get)[::-1]

    entries = []
    dict_names = sorted(list(kwargs.keys()))
    header_str = 'Avg scores: '
    for dn in dict_names:
        assert isinstance(kwargs[dn], dict)
        header_str += '{}: {:.2f} '.format(dn, mean_dict(kwargs[dn]))
    for k in topic_keys:
        score_str = []
        for dn in dict_names:
            # score_str.append('{} {:.2f}'.format(dn, kwargs[dn][k]))
            score_str.append('{:.2f}'.format(kwargs[dn][k]))
        score_str = ', '.join(score_str)
        entries.append('T{} [{}] '.format(k, score_str) + ', '.join(topic_json[k]))

    msg = header_str + '\n' + '\n'.join(entries)
    print(msg)
    return msg


# 上传操作分两种
# 1 直接可以输入的 send_keys(),输入上传的全路径即可
# 2 window 控件导致的上传 selenium 操作不了
# selenium 操作的都是 h5 相关的元素
#  用 python 的 pywin32 和 spy++
#与 window 系统交互的
#适用 ie firefox chrome
# pip install pywin32
import win32gui
import win32con

def upload(file_path):
    dialog = win32gui.FindWindow("#32770","打开")# 找到一级窗口
    #找到窗口
    comboBoxEx32 = win32gui.FindWindowEx(dialog,0,"ComboBoxEx32",None)

    comboBox = win32gui.FindWindowEx(comboBoxEx32,0,"ComboBox",None)

    edit = win32gui.FindWindowEx(comboBox,0,"Edit",None)
    button = win32gui.FindWindowEx(comboBox,0,"Button",None)

    file_path = ""
    win32gui.SendMessage(edit,win32con.VM_SETTEXT,None,file_path)
    import time
    time.sleep(5)
    win32gui.SendMessage(dialog,win32con.VM_COMMOD,1,button)


import os
from socials.config import settings
os.environ['env_mode'] = settings.env_mode
import time
from socials.apps.bots_tasks.models import *
from socials.apps.bots_tasks.enums import *
from socials.apps.bots_tasks.main import get_bot_tasks, process_bot_task
from socials.logging import lgd,lgw,lge

time_to_sleep = 3

def process_tasks(
    include_selenium_tasks = False,
    process_tasks_per_cycle = 10
):
    lgd('** Run process youtube tasks **')
    try:
        bot_tasks_query = BotTasksSearchQuery(
            platform = PlatformEnum.yt,
            status = BotTaskStatusEnum.running,
            is_active=True,
            include_hidden=True,
            limit=process_tasks_per_cycle,
            include_selenium_tasks=include_selenium_tasks,
            sort_by_updated_date=SortingOrder.ascending
        )
        bot_tasks_search: BotTasksSearch = get_bot_tasks(
            query = bot_tasks_query
        )
        for bot_task in bot_tasks_search.bot_tasks:
            process_bot_task(
                bot_task
            )
    except Exception as e:
        lge(f'error while run process tasks {e}')
        pass
    time.sleep(time_to_sleep)

if __name__ == '__main__':
    while True:
        process_tasks()


import requests
from flask import Flask, render_template, request, redirect, url_for, flash
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

app.config['DEBUG'] = True

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///weather.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SECRET_KEY'] = 'key'

db = SQLAlchemy(app)

class City(db.Model):

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50), nullable=False)

def get_weather_data(city):

    url = f'https://api.openweathermap.org/data/2.5/weather?q={ city }&units=imperial&appid=31711e4776520f7cce0f2f4e47b7ae24'

    req = requests.get(url).json()
    return req

@app.route('/')
def index_get():

    cities = City.query.all()

    weather_data =[]

    for city in cities:

        r = get_weather_data(city.name)

        weather = {
            'city' : city.name,
            'temperature' : str(round(int(r['main']['temp']))),
            'description' : r['weather'][0]['description'],
            'icon' : r['weather'][0]['icon'],
            'country' : r['sys']['country'],
            'feels_like' : str(round(int(r['main']['feels_like']))),
            'speed' : r['wind']['speed'],
        }

        weather_data.append(weather)

    return render_template('weatherapp.html', weather_data=reversed(weather_data))

@app.route('/', methods=['POST'])
def index_post():

    error_message = ''

    new_city = request.form.get('city')

    if new_city:
        existing_city = City.query.filter_by(name=new_city).first()

        if not existing_city:
            new_city_data = get_weather_data(new_city)

            if new_city_data['cod'] == 200:
                new_city_obj = City(name=new_city)

                db.session.add(new_city_obj)
                db.session.commit()

            else:
                error_message = 'City does not exist.'
        else:
            error_message = 'City already exists below.'

        if error_message:
            flash(error_message, 'error')

        else:
            flash('City added successfully.')

    return redirect(url_for('index_get'))

@app.route('/delete/<name>')
def delete_city(name):

    city = City.query.filter_by(name=name).first()

    db.session.delete(city)
    db.session.commit()

    flash(f'Successfully deleted { city.name }.', 'success')

    return redirect(url_for('index_get'))


# imports
from re import I
from opentrons import protocol_api

# metadata
metadata = {
    'protocolName': 'Distribute 20ul Water Each Well in 96w Plate.',
    'author': 'Harley King <harley.king@luminultra.com>',
    'description': 'Resuspending lyophilized pellet in 96w plate with 20ul water.',
    'apiLevel': '2.13'
}

##########################
# Calc heights for 50mL conical tubes
def fifty_ml_heights(init_vol, steps, vol_dec):
    vols = []
    heights = []
    # these values originate from Excel spreadsheet "Exp803..."
    b = 0
    m = 0.0024
    if init_vol > 51000:
        offset = 14  # model out of range; see sheet
    else:
        offset = 15  # mm Need to add offset to ensure tip reaches below liquid level
    for i in range(steps):
        x = init_vol-vol_dec*i
        vols.append(x)
        h = m*x+b
        h = h-offset
        if h < 17.5:  # If less than 5mL remain in 50mL tube, go to bottom for asp
            h = 2
            heights.append(h)
        else:
            heights.append(round(h, 1))
    return heights

def run(protocol: protocol_api.ProtocolContext):

    # EXPERIMENT OUTLINE
    # This experiment tests if dextran concentrations in the LyoReady Mastermix affects the qPCR results.
    # Dextran is in 50mL tube
    # LyoReady mastermix (or other) are added to 12, 1.5mL tubes chilled in temp module. EAch column is a concentration of Dextran. 12 different concentrations.
    # To these tubes dextran and water are added, mixed and aliquoted to PCR plate.

    # LABWARE
    tiprack300 = protocol.load_labware('opentrons_96_filtertiprack_200ul', '8')
    # tiprack20 = protocol.load_labware('opentrons_96_filtertiprack_20ul', '9')

    tempdeck = protocol.load_module('tempdeck', '1')
    plate = tempdeck.load_labware('bioer_96_aluminumblock_200ul')

    # alumBlock = protocol.load_labware('opentrons_24_tuberack_nest_1.5ml_snapcap', '2')
    water_rack = protocol.load_labware('opentrons_10_tuberack_nest_4x50ml_6x15ml_conical', '2')

    # reagent_rack = protocol.load_labware('opentrons_10_tuberack_nest_4x50ml_6x15ml_conical', '2')

    # PIPETTES
    p300 = protocol.load_instrument(
        'p300_single_gen2', 'left', tip_racks=[tiprack300]
    )

    # REAGENTS

    water = water_rack['A3'] # 50mL tube with 30mL NFW water
    waste = water_rack['A1'] # 5mL empty eppendorf tube for waste.

    # USER INPUT
    water_beg_vol = 30000 # water volume in ul

    # LISTS
    rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']

    # #### COMMANDS ######
    # 1.1 Add 20ul NFW to each well in plate

    # 1.1 Add 20ul of water to each well in plate
    h_list = fifty_ml_heights(30000, 12, 180)
    p300.pick_up_tip()
    p300.mix(3, 200, water.bottom(h_list[0]))
    for col in range(12):
        p300.aspirate(8*20+20, water.bottom(h_list[col])) # 8*14*excess = 120.96ul
        for row in rows:
            dest = plate[row+str(col+1)]
            p300.dispense(20, dest.bottom(3))
        p300.dispense(20, waste.center())
        p300.blow_out()
        p300.touch_tip(waste, speed=50, v_offset=-5)

    p300.drop_tip()

Exp800.18 LyoBead Programs on OT-2/Exp800.18.20 Add 20ul water to each well in 96w plate.py


import matplotlib.animation as animation
import matplotlib.pyplot as plt
import numpy as np

# creating a blank window
# for the animation
fig = plt.figure()
axis = plt.axes(xlim=(-10, 10), ylim=(-10, 10))

coordinates = np.array([
    [0, 0],
    [8, -4],
    [-2, 9],
    [-5, -7],
])
plt.scatter(coordinates[:, 0], coordinates[:, 1])
plt.axis('off')
plt.show()


import json
import pickle

from dataclasses import is_dataclass, fields
from ruka.util.json import xdumpj, xloadj, XJSONSerializable
from ruka.util.migrating import Migrating, State, validate_state
from ruka_os import distributed_fs_v2 as dfs

class SavedByRemotePath(Migrating):
    """
    Use this class if you wish to save something by DFS path rather than by
    value.

    - When saved through serialization mechanisms, such as pickle or
      ruka.util.json.xdump, only the remote path is saved.
    - When saved through .save() method, the "real" object contents are saved.

    The real object contents can be saved through pickling (default),
    ruka.util.json.xdump (see SavedByRemotePathToJSON), or in a completely
    custom way.

    Usage:

    1. Derive from SavedByRemotePath.

            @dataclass
            class BPL(SavedByRemotePath):
                '''
                A [b]ig [p]ersistent [l]ist.
                '''

                remote_path: str
                l: List[float]

       SavedByRemotePath is derived from Migratable, so you automatically get
       that functionality too.

    2. Create object.

            my_list = BPL(
                remote_path='//data/my_list.bpl.pickle',
                l=list(range(999999))
            )

       At this point, you can ignore the functionality of SavedByRemotePath
       completely, and use my_list as a regular dataclass.

    3. Save object to DFS.

            my_list.save()

    4. Load object from DFS.

            my_list_copy = BPL.load('//data/my_list.bpl.pickle')

    5. By default, save() and load() use pickling to save/load real contents.
       This can be overridden by inheriting from SavedByRemotePathToJSON
       instead of SavedByRemotePath:

            @dataclass
            class BPL(SavedByRemotePathToJSON):
                ...  # same as above

            my_list = BPL([1, 2, 3], '//data/my_list.bpl.json')
            my_list.save()
            my_list_copy = BPL.load('//data/my_list.bpl.json')
            my_list_copy = xloadr('//data/my_list.bpl.json')

        Note that when you derive from SavedByRemotePathToJSON, you can use
        xloadr() too.

        Also note, that you can edit '//data/my_list.bpl.json' by hand.

    6. You can also override load() and save() methods completely to not
       pickle, not JSON, but a custom format:

        @dataclass
        class JPEGImage(SavedByRemotePath):
            remote_path: str
            img: np.ndarray  # [H, W, 3]

            def save(self):
                dfs.upload_bytes(img2jpg(self.img), self.remote_path)

            @classmethod
            def load(cls, remote_path: str) -> JPEGImage:
                return JPEGImage(
                    remote_path=remote_path,
                    img=jpg2img(dfs.download_bytes(remote_path))
                )

       This image will be then saved by reference (i.e. by remote_path)
       instead of by value.
    """
    def __new__(cls, *args, **kwargs):
        assert is_dataclass(cls)
        assert 'remote_path' in {f.name for f in fields(cls)}
        return Migrating.__new__(cls)

    def xjson_getstate(self) -> State:
        if not isinstance(self.remote_path, str):
            raise RuntimeError(
                f'Cannot serialize {type(self)}, remote_path is {self.remote_path}')
        return {'remote_path': self.remote_path, 'class': self.xjson_getclass()}

    def xjson_setstate(self, state: State):
        validate_state(state, type(self))
        obj = self.load(state['remote_path'])
        assert type(obj) is type(self)
        for f in fields(self):
            setattr(self, f.name, getattr(obj, f.name))

    def save(self):
        if not isinstance(self.remote_path, str):
            raise RuntimeError(
                f'Cannot serialize {type(self)}, '
                f'remote_path is {self.remote_path}')
        state = Migrating.xjson_getstate(self)
        dfs.upload_bytes(pickle.dumps(state), self.remote_path)

    @classmethod
    def load(cls, remote_path: str) -> 'SavedByRemotePath':
        state = pickle.loads(dfs.download_bytes(remote_path))
        obj = cls.__new__(cls)
        Migrating.xjson_setstate(obj, state)
        return obj

class SavedByRemotePathToJSON(SavedByRemotePath):
    def save(self):
        if not isinstance(self.remote_path, str):
            raise RuntimeError(
                f'Cannot serialize {type(self)}, '
                f'remote_path is {self.remote_path}')
        state = Migrating.xjson_getstate(self)
        state['class'] = self.xjson_getclass()
        state = {k: xdumpj(v) for k, v in state.items()}
        dfs.upload_str(json.dumps(state), self.remote_path)

    @classmethod
    def load(cls, remote_path: str) -> 'SavedByRemotePathToJSON':
        state = json.loads(dfs.download_str(remote_path))
        state = {k: xloadj(v) for k, v in state.items()}
        cls = XJSONSerializable.xjson_loadclass(state['class'])
        obj = cls.__new__(cls)
        Migrating.xjson_setstate(obj, state)
        return obj


# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
import os
import re
import sys

sys.path.insert(0, os.path.abspath('..'))
sys.path.append(os.path.abspath('extensions'))

# -- Project information -----------------------------------------------------

project = 'steam community market'
copyright = '2020, offish'
author = 'offish'

with open('../steam_community_market/__init__.py') as f:
    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', f.read(), re.MULTILINE).group(1)

# The full version, including alpha/beta/rc tags
release = version

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'builder',
    'sphinx.ext.autodoc',
    'sphinx.ext.extlinks',
    'sphinx.ext.intersphinx',
    'sphinx.ext.napoleon',
    'details',
    'exception_hierarchy'
]

sphinx_mapping = {
    'py': ('https://docs.python.org/3', None)
}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

master_doc = 'index'

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

autodoc_member_order = 'bysource'

html_theme_options = {
    'description': 'by offish',
    'github_user': 'offish',
    'github_repo': 'steam_community_market',
    'github_button': 'true',
    'github_banner': 'true',
    'fixed_sidebar': 'true',
    'donate_url': 'https://steamcommunity.com/tradeoffer/new/?partner=293059984&token=0-l_idZR',
}

# Skal endres senere.
"""
{
    # Under skal flyttes til _static/custom.css

    'caption_font_size': '20px',
    'base_bg': '#282A36',  # bakgrunn
    'code_font_family': "'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace'",
    'link': '#5EBFE4',  # alle link farger
    'sidebar_link': '#D764A7',
    'body_bg': '#191A21',
    'highlight_bg': '#383A59',
    'viewcode_target_bg': '#ffd700',
    'code_text': '#B380CF',  # etter class og funksjon navn
    'body_text': '#F8F7F0',  # vanlig tekst og overskrifter
    'footer_text': '#BD93F9',  # footer text farge (bra)
    'gray_2': '#282A36',  # code embeds bra
    'link_hover': '#3CD872', # link hover farge bra
    'narrow_sidebar_bg': '#191A21', # når sidebar er mobilview (bra)
    'narrow_sidebar_link': '#D764A7',  # link farger i mobilview
    'pre_bg': '#343746',  # code blocks bla
    'sidebar_header': '#47CF5A',  # navigation, quick search og donate support
    'sidebar_search_button': '#F7F7F1', # søk button
    'sidebar_text': '#F6F3ED',  # by offish og contents
}
"""


# Ty Ridings
# CSCI 101 - Section D
# Python Lab 5B

years = int(input('Input the number of years to print in the reforestation table: \nYEARS> '))
x = 2500
i = 0

print('OUTPUT\t Year\t # Acres in Forest\t % of Forest')

while i < years:
    perc = (x / 14000)*100
    print('OUTPUT\t %d\t %d\t\t\t %0.2f' % (i,x,perc))
    x = (x * 1.02)
    i = i + 1

# Extra Credit Section
while perc < 100.00:
    if x < 14000:
        x = (x * 1.02)
        i = i + 1
    else:
        print('OUTPUT\t It will take',i,'years to restore the forest to 100% coverage')
        break


# 입력으로 Base64 Encoding 된 String 이 주어졌을 때, 해당 String 을 Decoding 하여, 원문을 출력하는 프로그램을 작성하시오.

from ast import Pass
from base64 import b64decode

T = int(input())
result = []
for i in range(T):
    string = b64decode(input()).decode("UTF-8")
    result.append(string)
for i in range(T):
    print(f'#{i+1} {result[i]}')

# 모듈쓰면 컴파일 오류남
T = int(input())
decode = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/']
result = []
for tc in range(1,1+T):

    # 입력받는 글자(인코딩 된 상태)
    word = list(input())

    # 글자의 길이
    length = len(word)

    # word의 글자 -> 표1에 따라 숫자로 변환 -> 이진수로 변환 -> 하나의 res로 만들기
    res = ''

    for q in range(length):

        # word -> 표1에 따라 숫자로 변환
        num = decode.index(word[q])

        # num -> 이진수로 변환
        # int형태로 더하면 단순 숫자의 합이 나오므로 str로 변환하고 앞의 0b 제거
        bin_num = str(bin(num)[2:])

        # bin_num의 길이가 6보다 작으면 앞에 0 붙여주기
        # 1을 이진수로 바꾸면 1로 나옴(필요한 값은 000001임)
        while len(bin_num) < 6:
            bin_num = '0' + bin_num

        res += bin_num

    # 문제에서 구하고자 하는 원래 문장
    r = ''

    # 글자의 길이 * 6에서 8비트씩 자름
    for w in range(length*6//8):

        # 8비트씩 자르기
        # 자른 값을 10진수로 변환
        e = int(res[w*8:w*8+8],2)

        # 아스키코드의 값을 r에 추가
        r += chr(e)
    result.append(r)
for j in range(T):
    print(f'#{j+1} {result[j]}')


#!/usr/bin/python3
# -*-coding:utf-8
import time
import numpy as np
import pandas as pd

def log_transformation(df):
    """
    Perform Log Transformation on features, which max value is > 100 (NOT LOGICAL!)
    """
    for col in df.columns:
        if 'label' in col:
            pass
        elif df[col].dtypes == np.int64 and df[col].max() > 100:
            print('Perform Transformation on column: ', col)
            df['log_{}'.format(col)] = np.log(df[col] + 1) # smoothing
            df.drop(col, axis = 1, inplace = True)
        elif df[col].dtypes == np.float64 and df[col].max() > 100:
            print('Perform Log Transformation on column: ', col)
            df['log_{}'.format(col)] = np.log(df[col] + 1) # smoothing
            df.drop(col, axis = 1, inplace = True)
    return df

def merge_features(dataType, airQuality):
    """
    Merge datetime features and air quality features into a dataframe

    params: dataType: str = ['train', 'test']
    params: airQuality: str = ['PM2.5', 'PM10']
    return: dataframe
    """
    df_datetime = pd.read_csv('../feature/london/{}/{}/datetime_features.csv'.format(dataType, airQuality))
    df_airquality = pd.read_csv('../feature/london/{}/{}/air_quality_features.csv'.format(dataType, airQuality))
    df = pd.merge(df_airquality, df_datetime, on=['station_id', 'utc_time'])

    return df

if __name__ == "__main__":
    s = time.time()

    inputPair = [(dataType, airQuality) for dataType in ['train', 'test'] for airQuality in ['PM2.5', 'PM10']]
    for pair in inputPair:
        print('Merging {} {} dataset...'.format(pair[1], pair[0]))
        df = merge_features(pair[0], pair[1])
        df.to_csv('../feature/london/{}/{}/all_features.csv'.format(pair[0], pair[1]), index=False)

    e = time.time()
    print('Process time: {:.2f} secs'.format(e-s))


import pytest
from unittest import TestCase
from PIL import Image

import sys, os, shutil, uuid
myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath + "/../src/makePDF/") # move to the src directory

import main

class TestMakePDF(TestCase):
    def test_make_pdf_wrong_file(self):
        with pytest.raises(IOError) as e_info:
            main.make_pdf_from_images(["/not/here.png"], "foo.pdf")
            assert e_info == "Something went wrong, I can't access the file at /not/here.png"

    def test_make_pdf(self):
        main.make_pdf_from_images(
        ['tests/images/travis-pride.png',
        'tests/images/travis-pride-sans-alpha.png'],
        output = "out.pdf"
        )


#세 자리 수를 곱해 만들 수 있는 가장 큰 대칭수(palindrome: 앞에서부터 읽을 때나 뒤에서부터 읽을 때나 모양이 같은 수)

def isPalindrome(num):
    s = str(num)
    start = 0
    end = len(s) - 1
    return s == s[::-1]

maxPalindrome = 0

for i in range(999,111,-1):
    for j in range(999,111,-1):
        k = i * j
        if maxPalindrome != 0 and k < maxPalindrome:
            break
        if isPalindrome(k):
            maxPalindrome = max(maxPalindrome, k)
            break;

print(maxPalindrome)


"""
Дана строка, состоящая из слов, разделенных пробелами. Определите, сколько в ней слов.
"""

str = input()

words = 0
pos = 0
while pos != -1:
    pos = str.find(" ", pos+1)
    words += 1

print(words)