import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gio, Gdk, GLib
import threading, time, math, cairo, inspect, os.path
from collections import deque

from . import LineChart, StatLine
from ...settings import Settings

class NetbandMonitorWindow(Gtk.Window):
    __gtype_name__ = 'NetbandMonitorWindow'

    class ChartUpdater(threading.Thread):
        def __init__(self, window, net_monitor, *args, **kwargs):
            super(NetbandMonitorWindow.ChartUpdater, self).__init__(*args, **kwargs)
            self.event = threading.Event()
            self.window = window
            self.max_up = -1
            self.max_down = -1
            self.total_up = 0
            self.total_down = 0
            self.averages_update_interval = 2
            self.last_averages_update = 0

        def set_interval(self, interval):
            self.interval = interval

        def format_value(self, bytes):
            mult = 8 if self.window.use_bits else 1
            unit = 'b' if self.window.use_bits else 'B'
            unit_prefix_base = 1024 if self.window.use_binary_prefix else 1000
            unit_prefix_names = ["", "Ki", "Mi", "Gi"] if self.window.use_binary_prefix else ["", "k", "m", "g"]
            n_fold = 0 if bytes == 0 else min(int(math.log(bytes * mult, unit_prefix_base)), 3)
            bytes_scaled = float(bytes) / (unit_prefix_base ** n_fold)
            return "{:4.1f}{}{}".format(bytes_scaled, unit_prefix_names[n_fold], unit)

        def run(self):
            self.running_since = time.time()
            self.last_update = self.running_since
            while True:
                self.event.wait(self.interval)
                if self.event.is_set():
                    self.event.clear()
                    break
                if len(self.window.chart.lines) != 2:
                    self.window.chart.lines.append(LineChart.Line([1, 0, 0, 1]))
                    self.window.chart.lines[0].data.extend([0 for x in range(10)])
                    self.window.chart.lines.append(LineChart.Line())
                    self.window.chart.lines[1].data.extend([0 for x in range(10)])
                now = time.time()
                delta_time = now - self.last_update
                (up, down, up_per_sec, down_per_sec, record_up_per_sec, record_down_per_sec) = (0, 0, 0, 0, 0, 0)
                self.window.net_monitor.interface_stats_delta_lock.acquire()
                for iface in self.window.net_monitor.interface_delta_per_sec:
                    ifacestats = self.window.net_monitor.interface_delta_per_sec[iface]
                    n = 0
                    for stat in ifacestats:
                        if stat.time > self.last_update:
                            up_per_sec += stat.count.tx
                            down_per_sec += stat.count.rx
                            n += 1
                    if n > 0:
                      up_per_sec /= n
                      down_per_sec /= n
                    ifacestats = self.window.net_monitor.interface_delta[iface]
                    for stat in ifacestats:
                        if stat.time > self.last_update:
                            up += stat.count.tx
                            down += stat.count.rx
                    record = self.window.net_monitor.interface_delta_per_sec_records[iface]
                    if record.rx > record_down_per_sec:
                        record_down_per_sec = record.rx
                    if record.tx > record_up_per_sec:
                        record_up_per_sec = record.tx
                self.window.net_monitor.interface_stats_delta_lock.release()
                # Total stats
                # - Download
                if down > 0:
                    self.total_down += down
                    self.window.stat_total.statistics[0].value = self.format_value(self.total_down)
                # - Upload
                if up > 0:
                    self.total_up = up
                    self.window.stat_total.statistics[1].value = self.format_value(self.total_up)

                # Instant stats
                # - Download
                down_formatted = self.format_value(down_per_sec) + "/s"
                # - - Max
                if down_per_sec > self.max_down:
                    self.max_down = down_per_sec
                    self.window.stat_download.statistics[2].value = down_formatted
                # - - Current
                self.window.stat_download.statistics[0].value = down_formatted
                # - Upload
                up_formatted = self.format_value(up_per_sec) + "/s"
                # - - Max
                if up_per_sec > self.max_up:
                    self.max_up = up_per_sec
                    self.window.stat_upload.statistics[2].value = up_formatted
                # - - Current
                self.window.stat_upload.statistics[0].value = up_formatted

                # Averages
                if now - self.last_averages_update > self.averages_update_interval:
                    self.last_averages_update = now
                    self.window.stat_download.statistics[1].value = \
                        self.format_value(self.total_down / (now - self.running_since)) + "/s"
                    self.window.stat_upload.statistics[1].value = \
                        self.format_value(self.total_up / (now - self.running_since)) + "/s"

                # Chart
                viz_maxdown = self.window.settings.status_icon_max_download_per_sec or \
                              self.window.settings.bandwidth_max_download_per_sec or \
                              record_down_per_sec or 1
                viz_maxup = self.window.settings.status_icon_max_upload_per_sec or \
                            self.window.settings.bandwidth_max_upload_per_sec or \
                            record_up_per_sec or 1
                viz_max = max(viz_maxup, viz_maxdown)
                self.window.chart.lines[0].data.append(down)
                self.window.chart.lines[1].data.append(up)
                self.window.chart.data_max = viz_max
                if len(self.window.chart.lines[0].data) > 10:
                    self.window.chart.lines[0].data.popleft()
                    self.window.chart.lines[1].data.popleft()
                self.window.chart.queue_draw()

                # Status icon (tray)
                if self.window.status_icon_surface is None:
                    self.window.status_icon_surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, 22, 22)
                if self.window.status_icon_context is None:
                    self.window.status_icon_context = cairo.Context(self.window.status_icon_surface)
                    self.window.status_icon_context.identity_matrix()
                    self.window.status_icon_context.set_line_width(0)
                ctx = self.window.status_icon_context
                ctx.set_source_rgb(*self.window.settings.status_icon_bgcolor)
                ctx.rectangle(0, 0, 22, 22)
                ctx.fill()
                ctx.set_source_rgb(*self.window.settings.status_icon_download_color)
                h = (down_per_sec / float(viz_max)) * 20
                ctx.rectangle(2, 1+(20-h), 9, h)
                ctx.fill()
                ctx.set_source_rgb(*self.window.settings.status_icon_upload_color)
                h = (up_per_sec / float(viz_max)) * 20
                ctx.rectangle(13, 1+(20-h), 9, h)
                ctx.fill()
                self.window.status_icon.set_from_pixbuf(Gdk.pixbuf_get_from_surface(
                    self.window.status_icon_surface, 0, 0, 22, 22))

                self.last_update = now

        def stop(self):
            self.event.set()

    def __init__(self, net_monitor, *args, **kwargs):
        super(NetbandMonitorWindow, self).__init__(*args, **kwargs)
        self.set_size_request(240, 40)
        self.settings = Settings()
        self.net_monitor = net_monitor
        self.screen = self.get_screen()
        self.display = self.screen.get_display()
        visual = self.screen.get_rgba_visual()
        self.transparency = False
        if visual and self.screen.is_composited():
            self.set_visual(visual)
            self.transparency = True
        else:
            sys.stderr.write('System doesn\'t support transparency\n')
            self.set_visual(self.screen.get_system_visual())
        css_provider = Gtk.CssProvider()
        css_provider.load_from_data(b'''
        .transparencible-win decoration {
            transition: box-shadow .5s linear;
        }
        .transparencible-win.transparent decoration {
            box-shadow: 0 3px 9px 1px rgba(0, 0, 0, 0), 0 0 0 1px rgba(255, 0, 0, 0);
        }
        .transparencible-win .titlebar, .transparencible {
            transition: opacity .5s linear;
        }
        .transparencible-win.transparent .titlebar, .transparent .transparencible {
            opacity: 0;
        }
        .transparencible-win.background {
            transition: background .5s linear;
        }
        .transparencible-win.background.transparent {
            background: transparent;
        }
        .transparencible-win.transparent .titlebar {
            opacity: 0;
        }
        .window-frame {
        box-shadow: none;
        margin: 0;
        }
        decoration {  }
        ''')
        self.context = self.get_style_context()
        self.context.add_provider_for_screen(self.screen, css_provider, Gtk.STYLE_PROVIDER_PRIORITY_USER)
        self.context.add_class('transparencible-win')

        self.vbox = Gtk.VBox()

        self.stat_download = StatLine("Download", 3)
        self.vbox.pack_start(self.stat_download, False, False, 0)
        self.stat_upload = StatLine("Upload", 3)
        self.vbox.pack_start(self.stat_upload, False, False, 0)
        for s in [self.stat_download.statistics, self.stat_upload.statistics]:
            s[0].title = "Current"
            s[1].title = "Average"
            s[2].title = "Max"

        self.stat_today = StatLine("Today", 2)
        self.vbox.pack_start(self.stat_today, False, False, 0)
        self.stat_total = StatLine("Total", 2)
        self.vbox.pack_start(self.stat_total, False, False, 0)
        for s in [self.stat_today.statistics, self.stat_total.statistics]:
            s[0].title = "Download"
            s[1].title = "Upload"

        self.chart = LineChart()
        self.vbox.pack_start(self.chart, True, True, 0)

        self.add(self.vbox)

        icontheme = Gtk.IconTheme.get_default()
        icontheme.append_search_path(os.path.join(os.path.dirname(inspect.stack()[0][1]), 'icons'))

        self.bar = Gtk.HeaderBar(title="Netband Monitor")
        self.bar.set_has_subtitle(False)
        #self.bar.set_subtitle("Monitoring...")
        self.bar.set_show_close_button(True)

        menubutton = Gtk.ToggleButton()
        menubutton.set_image(Gtk.Image.new_from_icon_name('emblem-system-symbolic', Gtk.IconSize.SMALL_TOOLBAR))
        menubutton.connect('toggled', lambda btn: self.popover.show_all() if btn.get_active() else self.popover.hide())
        self.popover = Gtk.Popover()
        self.popover.connect('closed', lambda popover: menubutton.set_active(False))
        self.popover.set_position(Gtk.PositionType.BOTTOM)
        self.popover.set_relative_to(menubutton)
        box = Gtk.VBox()
        box.set_spacing(5)
        unitbuttonbox = Gtk.ButtonBox(Gtk.Orientation.HORIZONTAL)
        unitbuttonbox.set_layout(Gtk.ButtonBoxStyle.EXPAND)
        self.bytes_button = Gtk.ToggleButton(label='')
        self.bits_button = Gtk.ToggleButton(label='')
        def bits_toggle(btn):
            if btn.get_active():
                self.use_bits = (btn == self.bits_button)
        for b in [self.bytes_button, self.bits_button]:
            b.get_child().set_halign(Gtk.Align.FILL)
            b.get_child().set_justify(Gtk.Justification.CENTER)
            b.connect('toggled', bits_toggle)
            unitbuttonbox.add(b)
        unitbuttonbox.set_property('margin', 5)
        unitbuttonbox.set_property('margin-bottom', 0)
        box.add(unitbuttonbox)
        self.binaryprefix_checkbutton = Gtk.CheckButton('')
        def binary_toggle(btn):
            self.use_binary_prefix = btn.get_active()
        self.binaryprefix_checkbutton.connect('toggled', binary_toggle)
        self.use_binary_prefix = True
        self.use_bits = False
        box.add(self.binaryprefix_checkbutton)

        box.add(Gtk.Separator(orientation=Gtk.Orientation.HORIZONTAL))

        box.add(Gtk.Label("Refresh rate"))
        refresh_rate_grid = Gtk.Grid()
        self.refresh_rate_label = Gtk.Label()
        self.refresh_rate_label.set_label('5000ms')
        self.refresh_rate_label.set_size_request(self.refresh_rate_label.get_allocation().width, -1)
        self.refresh_rate_scale = Gtk.Scale.new(Gtk.Orientation.HORIZONTAL,
            Gtk.Adjustment.new(100, 100, 5000, 50, 50, 0))
        self.refresh_rate_scale.set_draw_value(False)
        self.refresh_rate_scale.set_has_origin(False)
        self.refresh_rate_scale.set_round_digits(0)
        self.refresh_rate_scale.set_hexpand(True)
        self.refresh_rate_scale.connect('value-changed', lambda scale:
            self.refresh_rate_label.set_label(str(int(scale.get_value())) + "ms"))
        self.refresh_rate_scale.connect('value-changed', lambda scale:
            self.chart_updater.set_interval(float(int(scale.get_value()))/1000))
        refresh_rate_grid.attach(self.refresh_rate_scale, 0, 0, 1, 1)
        refresh_rate_grid.attach(self.refresh_rate_label, 1, 0, 1, 1)

        box.add(refresh_rate_grid)

        checkbutton = Gtk.CheckButton("cc")
        box.add(checkbutton)
        self.popover.add(box)
        #self.popover.set_size_request(40, 40)
        self.bar.pack_end(menubutton)

        pinbutton = Gtk.ToggleButton()
        pinbutton.set_image(Gtk.Image.new_from_icon_name('pin-window-symbolic', Gtk.IconSize.SMALL_TOOLBAR))
        self.get_style_context().get_color(Gtk.StateFlags.NORMAL)
        pinbutton.connect('toggled', lambda btn: self.set_keep_above(btn.get_active()))
        self.bar.pack_end(pinbutton)

        self.set_titlebar(self.bar)

        self.status_icon = Gtk.StatusIcon()
        self.status_icon_surface = None
        self.status_icon_context = None

        self.chart_updater = self.ChartUpdater(self, self.net_monitor)
        self.chart_updater.set_interval(0.1)
        self.chart_updater.start()

        self.connect('delete-event', lambda x, y: self.chart_updater.stop())

    @property
    def use_binary_prefix(self):
        return self.binaryprefix_checkbutton.get_active()

    @use_binary_prefix.setter
    def use_binary_prefix(self, use):
        self.binaryprefix_checkbutton.set_active(use)
        self.bytes_button.get_child().set_label("KiB, MiB\n(bytes)" if use else "kB, MB\n(bytes)")
        self.bits_button.get_child().set_label("Kib, Mib\n(bits)" if use else "kb, Mb\n(bits)")

    @property
    def use_bits(self):
        return self.bits_button.get_active()

    @use_bits.setter
    def use_bits(self, use):
        self.bits_button.set_active(use)
        self.bytes_button.set_active(not use)
        self.binaryprefix_checkbutton.set_label("Binary prefix (" + ("Kib, Mib)" if use else "KiB, MiB)"))

    @property
    def transparent(self):
        return self.context.has_class('transparent')

    @transparent.setter
    def transparent(self, value):
        if value:
            self.context.add_class('transparent')
        else:
            self.context.remove_class('transparent')


from client import ManagementClientSimple, InternalClient
from common import create_devices
import pytest

@pytest.mark.usefixtures("create_devices")
class TestDeleteDevice(InternalClient):
    def test_delete(self):
        #first pull existing device sets to get some existing device_id
        mc = ManagementClientSimple()
        devs = mc.get_all_devices()
        num_devs_base = len(devs)

        assert num_devs_base > 0

        existing = devs[0]

        #delete existing device set, verify it's gone
        rsp = self.delete_device(existing.device_id)
        assert rsp.status_code == 204

        devs = mc.get_all_devices()
        num_devs = len(devs)
        assert num_devs == num_devs_base - 1

        found = [x for x in devs if x.device_id == existing.device_id]
        assert len(found) == 0

        #delete nonexistent device set, verify nothing's missing
        rsp = self.delete_device('foobar')
        assert rsp.status_code == 204

        devs = mc.get_all_devices()
        assert len(devs) == num_devs


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#.--. .-. ... .... -. - ... .-.-.- .. -.

import datetime
import dateutil.parser
import uuid
import random
import json
import math

from flask import request, g
from bson.objectid import ObjectId
from jsonschema import validate, ValidationError
from mongoengine.queryset import queryset_manager
from mongoengine.fields import (URLField, DictField, BooleanField, StringField,
	ListField, DateTimeField)
from survaider.minions.helpers import HashId, Obfuscate, Uploads
from survaider.minions.contextresolver import current_user
from survaider.minions.attachment import Image as AttachmentImage
from survaider.user.model import User
from survaider.notification.signals import survey_response_notify
from survaider.survey.structuretemplate import survey_struct_schema
from survaider.survey.structuretemplate import starter_template
from survaider import db, app

class Survey(db.Document):
	added	   = db.DateTimeField(default = datetime.datetime.now)

	metadata	= db.DictField()
	structure   = db.DictField()
	attachments = db.ListField(db.ReferenceField(AttachmentImage))

	created_by  = db.ListField(db.ReferenceField(User))

	meta = {'allow_inheritance': True, 'strict': False}

	def __unicode__(self):
		return HashId.encode(self.id)

	@property
	def resolved_root(self):
		return self

	@property
	def cols(self):
		return [_['cid'] for _ in self.structure['fields']]

	@property
	def notification_hooks(self):
		rules = {}
		check = []
		for field in self.structure['fields']:
			if field.get('notifications', False) is True:
				store = []
				fieldType = field.get('field_type')
				if fieldType == 'rating':
					options = enumerate(field['field_options'].get('notifications', []))
					for i,option in options :
					 store.append("a_"+option)

				if fieldType == 'yes_no' or fieldType == 'single_choice':
					options = enumerate(field['field_options'].get('options', []))
					for i, option in options:
						if option.get('notify', False) is True:
							val = "a_{0}".format(i + 1)
							store.append(val)
							for j in range(0, 5):
								 if option.get("notify_{0}".format(j)):
									 store.append("a_{0}##{1}".format(i + 1, j ))

				if fieldType == 'group_rating':
					options = enumerate(field['field_options'].get('options', []))
					for i, option in options:
						if option.get('notify', False) is True:

							for j in range(0, 5):
								 if option.get("notify_{0}".format(j)):
									 store.append("a_{0}##{1}".format(i + 1, j ))

				rules[field['cid']] = store

		return rules

	@property
	def questions(self):
		return [[_['cid'], _['label']] for _ in self.structure['fields']]

	@property
	def expires(self):
		return self.metadata['expires'] if 'expires' in self.metadata else datetime.datetime.max

	@expires.setter
	def expires(self, value):
		self.metadata['expires'] = value

	@property
	def active(self):
		time_now = datetime.datetime.now()
		return all([
			not self.hidden,
			time_now <= self.expires,
			not self.paused,
			self.response_cap > self.obtained_responses,
		])

	@property
	def paused(self):
		return self.metadata['paused'] if 'paused' in self.metadata else False

	@paused.setter
	def paused(self, value):
		self.metadata['paused'] = value

	@property
	def response_cap(self):
		return self.metadata['response_cap'] if 'response_cap' in self.metadata else 2**32

	@property
	def obtained_responses(self):
		return ResponseAggregation(self).count

	@response_cap.setter
	def response_cap(self, value):
		self.metadata['response_cap'] = value

	@property
	def struct(self):
		ret = {}
		ret['fields'] = self.structure.get('fields',
										   starter_template['fields'])
		ret['screens'] = self.structure.get('screens',
										   starter_template['screens'])
		ret['links'] = self.structure.get('links', None)

		return ret

	@struct.setter
	def struct(self, value):
		try:
			validate(value, survey_struct_schema)
			self.structure.update(value)
		except ValidationError as e:
			raise TypeError('Struct value invalid' + str(e))

	#: DEPRECATED
	@property
	def gamified_enabled(self):
		return False

	@property
	def modified(self):
		return self.metadata['modified'] if 'modified' in self.metadata else self.added

	@property
	def img_uploads(self):
		return [_.repr for _ in self.attachments]

	@property
	def hidden(self):
		return self.metadata['hidden'] if 'hidden' in self.metadata else False

	@hidden.setter
	def hidden(self, value):
		self.metadata['hidden'] = value

	@property
	def units(self):
		dat = SurveyUnit.objects(referenced = self)
		return [_.repr for _ in dat if not _.hidden]

	@property
	def unit_count(self):
		return SurveyUnit.objects(referenced = self).count()

	@property
	def units_as_objects(self):
		dat = SurveyUnit.objects(referenced = self)
		return [_ for _ in dat if not _.hidden]

	def save(self, **kwargs):
		self.metadata['modified'] = datetime.datetime.now()
		super(Survey, self).save(**kwargs)

	@queryset_manager
	def objects(doc_cls, queryset):
		return queryset.filter()

	@queryset_manager
	def root(doc_cls, queryset):
		return queryset.filter(_cls = 'Survey')

	@property
	def providers(self):
		dat = ClientProviders.objects(parent_id = str(self.resolved_root))
		providers = dat[0].providers
		return providers

	#: DEPRECATION WARNING: WILL BE CHANGED.
	@property
	def repr(self):
		return {
			'id': str(self),
			'name': self.metadata['name'],
			'ext': self.metadata.get('external', []),
			'is_gamified': self.gamified_enabled,
			'uri_simple': '/survey/s:{0}/simple'.format(str(self)),
			'uri_game': '/survey/s:{0}/gamified'.format(str(self)),
			'uri_edit': '/survey/s:{0}/edit'.format(str(self)),
			'uri_responses': '/survey/s:{0}/analysis'.format(str(self)),
			'is_paused': self.paused,
			'is_active': self.active,
			'imgs': self.img_uploads,
			'units': self.units,
			'has_response_cap': self.response_cap,
			'has_obtained_responses': self.obtained_responses,
			'has_expired': self.expires <= datetime.datetime.now(),
			'expires': str(self.expires),
			'created_on': str(self.added),
			'last_modified': str(self.modified),
			'providers': self.providers
		}

	@property
	def repr_sm(self):
		return {
			'id': str(self),
			'meta': {
				'name': self.metadata['name'],
				'type': self._cls,
			},
			'status': {
				'paused': self.paused,
				'active': self.active,
				'response_cap': self.response_cap,
				'response_count': self.obtained_responses,
				'expired': self.expires <= datetime.datetime.now(),
				'unit_count': self.unit_count,
			},
			'access': {
				'accessible': [_.repr for _ in self.created_by],
				'editable': True, #: Root Survey
				'owner': True,
			},
			'info': {
				'expires': str(self.expires),
				'added': str(self.added),
				'modified': str(self.modified),
			},
			'providers': self.providers
		}

	@property
	def tiny_repr(self):
		return {
			'id':	   str(self),
			'name':	 self.metadata['name'],
			'active':   self.active,
		}

	#: DEPRECATED
	@property
	def render_json(self):
		#: DEPRECATION WARNING: This method will be removed.
		rt = {}
		rt['fields'] = []
		rt['game_title'] = ""
		rt['game_description'] = ""
		rt['game_footer'] = ""
		rt['survey_logo'] = ""
		rt['WARNING'] = "This Feature has been deprecated.\
			It will be removed in coming versions.\
			This API Call has been preserved for backwards\
			compatibility."
		return rt

	@property
	def render_deepjson(self):

		rt = {}
		cp = self.struct['fields']

		def logic(id_next):
			return {
				'va': id_next
			}

		for i in range(len(cp)):
			option_len = len(cp[i]['field_options'].get('options', []))
			for j in range(option_len):
				img = cp[i]['field_options']['options'][j].get('img_uri', None)
				if img is not None:
					cp[i]['field_options']['options'][j]['img_uri'] = Uploads.url_for_surveyimg(img)
			cp[i]['next'] = logic('end' if (i + 1) >= len(cp) else cp[i + 1]['cid'])

		rt['fields'] = cp
		rt['survey_title'] = self.struct['screens'][0]

		give_img = len(self.struct['screens']) >= 4 and len(self.struct['screens'][3]) > 1

		rt['survey_logo'] = Uploads.url_for_surveyimg(self.struct['screens'][3]) if give_img else False
		rt['survey_description'] = self.struct['screens'][1]
		rt['survey_footer'] = self.struct['screens'][2]
		return rt

class SurveyUnit(Survey):
	referenced = db.ReferenceField(Survey, required = True)
	unit_name  = db.StringField()

	def __init__(self, **kwargs):
		super(SurveyUnit, self).__init__(**kwargs)
		if self.referenced:
			self.structure = self.referenced.structure
			self.metadata.update(self.referenced.metadata)

	@property
	def units(self):
		"SurveyUnits cannot have Units."
		return None

	@property
	def resolved_root(self):
		return self.referenced.resolved_root

	@property
	def repr(self):
		return {
			'id': str(self),
			'unit_name': self.unit_name,
			'name': self.metadata['name'],
			'is_gamified': self.gamified_enabled,
			'uri_simple': '/survey/s:{0}/simple'.format(str(self)),
			'uri_game': '/survey/s:{0}/gamified'.format(str(self)),
			'uri_edit': '/survey/s:{0}/edit'.format(str(self)),
			'uri_responses': '/survey/s:{0}/analysis'.format(str(self)),
			'is_paused': self.paused,
			'is_active': self.active,
			'has_response_cap': self.response_cap,
			'has_obtained_responses': self.obtained_responses,
			'has_expired': self.expires <= datetime.datetime.now(),
			'expires': str(self.expires),
			'created_on': str(self.added),
			'last_modified': str(self.modified),
		}

	@property
	def repr_sm(self):
		return {
			'id': str(self),
			'rootid': str(self.resolved_root),
			'meta': {
				'name': self.unit_name,
				'rootname': self.resolved_root.metadata['name'],
				'type': self._cls,
			},
			'status': {
				'paused': self.paused,
				'active': self.active,
				'response_cap': self.response_cap,
				'response_count': self.obtained_responses,
				'expired': self.expires <= datetime.datetime.now(),
				'unit_count': False,
			},
			'access': {
				'accessible': [_.repr for _ in self.created_by],
				'editable': False, #: Unit Survey
				'owner': current_user() in self.resolved_root.created_by,
			},
			'info': {
				'expires': str(self.expires),
				'added': str(self.added),
				'modified': str(self.modified),
			}
		}

	@property
	def tiny_repr(self):
		return {
			'id':	   str(self),
			'name':	 self.unit_name,
			'active':   self.active,
		}

	@queryset_manager
	def objects(doc_cls, queryset):
		return queryset.filter()

class Response(db.Document):
	parent_survey   = db.ReferenceField(Survey)

	metadata		= db.DictField()
	responses	   = db.DictField()

	def __unicode__(self):
		return HashId.encode(self.id)

	def add(self, q_id, q_res, q_res_plain, q_unit_id = None):
		if q_id in self.parent_survey.cols:
			self.responses[q_id] = {'raw': q_res, 'pretty': q_res_plain, 'unit_id' : q_unit_id}
			self.metadata['modified'] = datetime.datetime.now()
			self.save()
		else:
			raise TypeError("Question ID is invalid")

		if q_id in self.parent_survey.notification_hooks:
			for hook in q_res.split('###'):
				if hook in self.parent_survey.notification_hooks[q_id]:
					survey_response_notify.send(self.parent_survey,
												response = self,
												qid = q_id,
												qres = hook)

	@property
	def added(self):
		return self.metadata['started'] if 'started' in self.metadata else datetime.datetime.min

	@property
	def response_sm(self):
		questions = dict(self.parent_survey.questions)
		res = []
		for k, v in self.responses.items():
			res.append({
				'id': k,
				'label': questions[k],
				'response': v['pretty']
			})

		return {
			'id': str(self),
			# 'meta': self.metadata,
			'parent_survey': self.parent_survey.repr_sm,
			'responses': res
		}

class ResponseSession(object):

	@staticmethod
	def start(survey_id, response_id):
		#: Payload: [Survey ID, Start Time, End Time, Finished?]
		expires = datetime.datetime.now() + datetime.timedelta(days=1)
		payload = {
			str(survey_id): [expires.isoformat(), str(response_id), False]
		}

		g.SRPL.update(payload)

	@staticmethod
	def get_running_id(survey_id):
		s_id = str(survey_id)
		if s_id in g.SRPL:
			res_id = g.SRPL[s_id][1]
			return res_id

	@staticmethod
	def is_running(survey_id):
		s_id = str(survey_id)
		if s_id in g.SRPL:
			return all([
				dateutil.parser.parse(g.SRPL[s_id][0]) > datetime.datetime.now(),
				g.SRPL[s_id][2] is False
			])

		return False

	@staticmethod
	def finish_running(survey_id):
		s_id = str(survey_id)
		if s_id in g.SRPL:
			del g.SRPL[s_id]

class ResponseAggregation(object):
	def __init__(self, survey):
		self.survey = survey

	def flat(self):
		responses = Response.objects(parent_survey = self.survey)

		cols = ["response_id", "added"] + self.survey.cols

		rows = []

		for response in responses:
			row = [str(response), str(response.added)]
			for qid in self.survey.cols:
				if qid in response.responses:
					row.append(response.responses[qid]['raw'])
				else:
					row.append(None)
			rows.append(row)

		return {
			"page": 0,
			"columns": cols,
			"questions": self.survey.questions,
			"rows": rows,
			"len": self.count,
			"survey_id": str(self.survey)
		}

	def nested(self):
		responses = Response.objects(parent_survey = self.survey)

		questions = self.survey.questions

		rows = []
		cols = [['Question', None]]
		ps = True

		for q in questions:
			row = []
			row.append(q[1])
			for response in responses:
				if ps:
					cols.append([str(response.id), str(response.added)])
				if q[0] in response.responses:
					row.append(response.responses[q[0]]['raw'])
				else:
					row.append(None)
			rows.append(row)
			ps = False

		return {
			"page": 0,
			"columns": cols,
			"questions": self.survey.questions,
			"rows": rows,
			"len": self.count,
			"survey_id": str(self.survey)
		}

	def csv(self):
		# Columns
		# Question ID | Question | Response ID 1 ... n
		responses = Response.objects(parent_survey = self.survey)
		questions = self.survey.questions

		header = ['question_id']
		for r in responses:
			header.append(str(r))

		yield ','.join(header)

		for q in questions:
			row = []
			row.append(q[0])
			row.append('"{0}"'.format(q[1]))
			for r in responses:
				row.append('"{0}"'.format(r.responses.get(q[0], '')['pretty']))
			yield ','.join(row)

	@property
	def count(self):
		return Response.objects(parent_survey = self.survey).count()
# Zurez
from bson.json_util import dumps

def d(data):
	return json.loads(dumps(data))

class AspectData(db.Document):
	"""doc string for Aspect"""
	name=db.StringField()
	provider=db.StringField()
	survey_id=db.StringField()
	value=db.StringField()
	meta = {'strict': False}

class IrapiData(object):
	"""docstring for IrapiData"""
	def __init__(self, survey_id,start,end,aggregate):
		self.sid = survey_id
		self.start = start
		self.end = end
		self.agg = aggregate

	def flag(self):
		dat = SurveyUnit.objects(referenced = self.sid)
		# return d(dat)
		js= [_.repr for _ in dat if not _.hidden]
		if len(js)!=0:
			children=[]
			for i in js:
				children.append(i['id'])
			return children
		else:
			return False

	def get_multi_data(self, aList):
		js=[]
		for i in aList:
			child_id= HashId.decode(i)
			# print (child_id)
			raw = Response.objects(parent_survey=child_id)
			raw_temp=[]
			for i in raw:
				temp_j=[]
				temp_j.append(i.responses)
				temp_j.append(i.metadata)
				raw_temp.append(temp_j)
			js= js + raw_temp

		return js

	def get_child_data(self,survey_id):

		raw=SurveyUnit.objects(id = survey_id)
		return raw

	def get_data(self):
		dat = Survey.objects(id = self.sid)
		flag= self.flag()

		if flag==False:
			raw= Response.objects(parent_survey=self.sid)
			raw_temp=[]
			for i in raw:
				temp_j=[]
				temp_j.append(i.responses)
				temp_j.append(i.metadata)
				raw_temp.append(temp_j)
			return raw_temp
		else:
			if self.agg=="true":

				"WIll return all the responses "

				raw = Response.objects(parent_survey = self.sid)
				raw_temp=[]
				for i in raw:
					temp_j=[]
					temp_j.append(i.responses)
					temp_j.append(i.metadata)
					raw_temp.append(temp_j)

				js = raw_temp + self.get_multi_data(flag)
				return js
			else:

				raw=Response.objects(parent_survey=self.sid)
				js= d(raw)
				return js

	def get_parent(self):
		raw = Survey.objects(id = self.sid)
		js = [_.repr_sm for _ in raw if not _.hidden]

		if 'rootid' in js[0]:
			return js[0]['rootid']
		else:
			return False

	def get_uuid_labels(self):
		raw=Survey.objects(id = self.sid)

		m = int(self.start)-1
		n = int(self.end)

		if "fields" in d(raw[0].structure):
			a= raw[0]
			if m==-1 and n==0:
				return a.structure['fields']

			else:
				return a.structure['fields'][m:n]

		return d(raw[0].structure['fields']) # fallback

	def survey_strct(self):
		try:
			raw=Survey.objects(id = HashId.decode(self.sid))
		except:
			raw=Survey.objects(id = self.sid)

		js=raw[0]['structure']['fields']
		# js=raw[0]

		return js

	def ret(self):
		try:
			raw=SurveyUnit.objects(referenced = self.sid)
			return d(raw)
		except:return "Errors"

class Leaderboard(db.Document):
	survey_ID = db.StringField()
	competitors = db.DictField()

class LeaderboardAggregator(object):
	def __init__(self, survey_id):
		self.sid = survey_id

	def getLeaderboard(self):
		raw_data = Leaderboard.objects(survey_ID = HashId.encode(self.sid))
		if len(raw_data) == 0:
			return None
		ordered_leaderboard_list = []
		unordered_leaderboard_dict = d(raw_data[0].competitors)

		ordered_scores_list = sorted(unordered_leaderboard_dict, key = unordered_leaderboard_dict.get, reverse=True)

		for i in ordered_scores_list:
			ordered_leaderboard_list.append([i, unordered_leaderboard_dict[i]])

		return ordered_leaderboard_list

class Insights(db.Document):
	survey_id = db.StringField()
	insights = db.DictField()

class InsightsAggregator(object):
	def __init__(self, survey_id):
		self.sid = survey_id

	def getInsights(self):
		raw_data = Insights.objects(survey_id = HashId.encode(self.sid))
		if len(raw_data) == 0:
			return None
		unordered_insight_dict = d(raw_data[0].insights)
		ordered_dates_list = sorted(unordered_insight_dict, key = lambda t: datetime.datetime.strptime(t, '%d-%m-%Y'), reverse=True)
		ordered_insights_list = []
		for i in ordered_dates_list:
			ordered_insights_list.append([i, list(unordered_insight_dict[i].values())])
		return ordered_insights_list

class Dashboard(IrapiData):
	"""docstring for Dashboard"""
	def __init__(self,survey_id):
		self.sid= survey_id

class WordCloudD(db.Document):
	"""docstring for WordCloud"""
	provider = db.StringField()
	survey_id = db.StringField()
	wc = db.DictField()

class Reviews(db.Document):
	provider = db.StringField()
	survey_id = db.StringField()
	rating = db.StringField()
	review = db.StringField()
	sentiment = db.StringField()
	review_identifier=db.StringField(unique=True)
	date_added=db.DateTimeField()
	review_link=db.StringField()
	datetime=db.DateTimeField()
	meta = {'strict': False}

class DataSort(object):
	"""docstring for DataSort"""

	def __init__(self,survey_id,uuid,aggregate):
		self.sid= survey_id
		self.uuid= uuid
		self.agg= aggregate
	def get_survey(self):
		survey=SurveyUnit.objects(referenced=self.sid)
		return d(survey)
	def get_response(self):
		response= db.response.find({"parent_survey":ObjectId(self.sid)})
		return d(response)
	def flag(self):
		# raw= db.survey.find({"_id":ObjectId(self.sid)})
		# js= d(raw)
		dat = SurveyUnit.objects(referenced = self.sid)
		js= [_.repr for _ in dat if not _.hidden]
		if len(js)!=0:
			children=[]
			for i in js:
				children.append(i['id'])
			return children
		else:
			return False

	def get_multi_data(self,aList):
		js=[]
		# return aLists
		for i in aList:
			i= HashId.decode(i)
			raw= db.response.find({"parent_survey":ObjectId(i)})
			js= js +d(raw)
		return js
	def get_child_data(self,survey_id):
		# raw= db.survey.find({"_id":ObjectId(self.sid)})
		raw=SurveyUnit.objects(referenced = self.sid)
		return d(raw)
	def get_data(self):
		dat = SurveyUnit.objects(referenced = self.sid)
		# return [_.repr for _ in dat if not _.hidden]
		flag= self.flag()
		# return flag
		if flag==False:
			raw= db.response.find({"parent_survey":ObjectId(self.sid)})
			js= d(raw)

			return js
		else:
			if self.agg=="true":
				"WIll return all the responses "
				raw= db.response.find({"parent_survey":ObjectId(self.sid)})
				js= d(raw)
				js = js+ self.get_multi_data(flag)
				return js
			else:
				raw= db.response.find({"parent_survey":ObjectId(self.sid)})
				js= d(raw)
				return js

	def get_uuid_label(self):
		"""labels: question text ; options ; etc"""
		#Extract the particular cid from the survey structure
		# raw_label=db.survey.find() #returns empty
		# raw_label=db.survey.find({"_id":ObjectId(self.sid)})
		try:
			raw_label=Survey.objects(id = HashId.decode(self.sid))
		except:
			raw_label=Survey.objects(id = self.sid)
		# return d(raw_label)

		aList= raw_label[0].structure['fields'] #A backup liseturn aList
		# return aList
		# aList= d(raw_label)
		# return aList
		for i in aList:
			if i['cid']==self.uuid:
				return i
				# Zurez

class Relation(db.Document):
	"""docstring for Relation"""
	survey_id = db.StringField()
	providers = db.ListField()
	parent_id = db.StringField()

class ClientAspects(db.Document):
	parent_id = db.StringField(required = True)
	aspects = db.ListField()

class ClientProviders(db.Document):
	parent_id = db.StringField(required = True)
	providers = db.ListField()

class TimedDash(db.Document):
	dash_value= db.StringField()
	time= db.DateTimeField(default = datetime.datetime.now)

# class JupiterData(db.Document):
#	 owner_aspects = db.DictField()
#	 units_aspects = db.DictField()
#	 last_updated = db.DateTimeField(default=datetime.datetime.now)
#	 survey_id = db.StringField()
#	 temp={}
#	 unit = {}
#	 def add(self,j_data,s_id):
#
#		 self.survey_id = HashId.encode(s_id)
#		 temp = j_data['owner_aspects']
#		 self.updateOwner(temp,self.survey_id)
#
#		 unit  = j_data['units_aspects']
#
#		 for key in unit :
#			 self.updateUnits(key,unit[key],self.survey_id)
#
#	 def updateOwner(self,data,s_id):
#
#		 ownerfinal = []
#		 ownerTime_temp = {}
#		 self.owner_aspects['overall_aspects'] = data['overall_aspects']
#
#		 self.owner_aspects['total_resp'] = data['total_resp']
#		 self.owner_aspects['providers'] = data['providers']
#		 self.owner_aspects['unified'] = data['unified']
#		 time = str(datetime.datetime.now().date())
#		 ownerTime_temp[time] = data['unified']
#
#		 ju_obj_temp1 = JupiterData.objects(survey_id = str(s_id))
#		 if len(ju_obj_temp1)-1 <0 :
#			 ownerfinal.append(ownerTime_temp)
#		 else :
#			 ju_obj_temp1 = JupiterData.objects(survey_id = str(s_id))
#			 ju_obj1 = ju_obj_temp1[len(ju_obj_temp1)-1]
#			 t=ju_obj1['owner_aspects']
#			 p = t['time_unified']
#			 ownerfinal = p
#			 ownerfinal.append(ownerTime_temp)
#		 self.owner_aspects['time_unified'] = ownerfinal
#
#	 def updateUnits(self,u_id,data,s_id):
#
#		 unitTemp = {}
#		 unitfinal = []
#		 unitTime_temp = {}
#		 unitTemp['overall_aspects'] = data['overall_aspects']
#		 unitTemp['total_resp'] = data['total_resp']
#		 unitTemp['providers'] = data['providers']
#		 unitTemp['unified'] = data['unified']
#		 time = str(datetime.datetime.now().date())
#		 unitTime_temp[time] = data['unified']
#
#		 ju_obj_temp1 = JupiterData.objects(survey_id = s_id)
#
#		 if len(ju_obj_temp1)-2 <0 :
#			 unitfinal.append(unitTime_temp)
#		 else :
#			 ju_obj1 = ju_obj_temp1[len(ju_obj_temp1)-2]
#			 t=ju_obj1['units_aspects']
#			 p = t[u_id]
#			 k = p['time_unified']
#			 unitfinal = k
#			 unitfinal.append(unitTime_temp)
#		 unitTemp['time_unified'] = unitfinal
#		 self.units_aspects[u_id] = unitTemp

class JupiterData(db.Document):
	owner_aspects = db.DictField()
	units_aspects = db.DictField()
	last_updated = db.DateTimeField(default=datetime.datetime.now)
	survey_id = db.StringField()
	temp={}
	unit = {}
	def update(self,j_data,s_id):

		self.survey_id = HashId.encode(s_id)
		temp = j_data['owner_aspects']
		self.updateOwner(temp,self.survey_id)

		unit  = j_data['units_aspects']

		for key in unit :
			self.updateUnits(key,unit[key],self.survey_id)

	def updateOwner(self,data,s_id):

		ownerfinal = []
		ownerTime_temp = {}
		self.owner_aspects['overall_aspects'] = data['overall_aspects']

		self.owner_aspects['total_resp'] = data['total_resp']
		self.owner_aspects['providers'] = data['providers']
		self.owner_aspects['unified'] = data['unified']
		time = str(datetime.datetime.now().date())
		ownerTime_temp[time] = data['unified']

		ju_obj_temp1 = JupiterData.objects(survey_id = str(s_id))
		if len(ju_obj_temp1)-1 < 0 :
			ownerfinal.append(ownerTime_temp)
		else :
			ju_obj_temp1 = JupiterData.objects(survey_id = str(s_id))
			ju_obj1 = ju_obj_temp1[len(ju_obj_temp1)-1]
			t = ju_obj1['owner_aspects']
			p = t['time_unified']
			ownerfinal = p
			ownerfinal.append(ownerTime_temp)
		self.owner_aspects['time_unified'] = ownerfinal

	def updateUnits(self,u_id,data,s_id):

		unitTemp = {}
		unitfinal = []
		unitTime_temp = {}
		unitTemp['overall_aspects'] = data['overall_aspects']
		unitTemp['total_resp'] = data['total_resp']
		unitTemp['providers'] = data['providers']
		unitTemp['unified'] = data['unified']
		time = str(datetime.datetime.now().date())
		unitTime_temp[time] = data['unified']

		ju_obj_temp1 = JupiterData.objects(survey_id = u_id)
		if len(ju_obj_temp1)-2 < 0 :
			unitfinal.append(unitTime_temp)
		else :
			ju_obj1 = ju_obj_temp1[len(ju_obj_temp1)-2]
			t=ju_obj1['units_aspects']
			p = t[u_id]
			k = p['time_unified']
			unitfinal = k
			unitfinal.append(unitTime_temp)
		unitTemp['time_unified'] = unitfinal
		self.units_aspects[u_id] = unitTemp

class AspectQ(db.Document):
	survey_id = StringField(required=True)
	base_url = URLField(required=True)
	unique_identifier = StringField(required=True, unique=True)
	time_review = DateTimeField(default='1980-01-01')
	last_update = DateTimeField(default='1980-01-01')

	meta = {'allow_inheritance': True}

	@property
	def repr(self):
		return {
			'id': str(self.pk),
			'access_url': self.base_url,
			'survey_id': self.survey_id,
			# 'children': self.children
		}

class ZomatoQ(AspectQ):
	provider = StringField(default="zomato")

class TripAdvisorQ(AspectQ):
	provider = StringField(default="tripadvisor")

class HolidayIQQ(AspectQ):
	provider = StringField(default="HolidayIQ")

class BookingQ(AspectQ):
	provider = StringField(default="booking")


# coding=UTF-8
import openpyxl
import os

def images_count(flag, names):
    return len([i for i in names if flag in i])

def statics(terrible_path):
    final_path = terrible_path.replace('terrible', 'final')
    names = os.listdir(terrible_path)
    total_terrible_M, total_terrible_W = 0, 0
    total_M, total_W = 0, 0
    values = [[
        '日期', '需要修正（男）', '总数（男）', '需要修正所占比例（男）', '需要修正（女）', '总数（女）',
        '需要修正所占比例（女）', '需要修正（男女合计）', '不需要修正（男女合计）', '总数（男女合计）',
        '需要修正所占比例（男女合计）'
    ]]
    for n in names:
        terrible_images_path = os.path.join(terrible_path, n)
        final_dir_path = os.path.join(final_path, n)
        terrible_images_names = os.listdir(terrible_images_path)
        terrible_M = images_count('_M_', terrible_images_names) // 5
        terrible_W = images_count('_W_', terrible_images_names) // 5
        total_terrible_M += terrible_M
        total_terrible_W += terrible_W
        final_images_names = os.listdir(final_dir_path)
        final_M = images_count('_M_', final_images_names) * 90
        final_W = images_count('_W_', final_images_names) * 90
        total_M += final_M
        total_W += final_W
        values.append([
            n.replace('2020', ''), terrible_M, final_M,
            str(round(terrible_M / final_M * 100, 2)) +
            '%' if final_M != 0 else 'inf', terrible_W, final_W,
            str(round(terrible_W / final_W * 100, 2)) +
            '%' if final_W != 0 else 'inf', terrible_M + terrible_W,
            final_M + final_W - terrible_M + terrible_W, final_M + final_W,
            str(round((terrible_M + terrible_W) /
                      (final_M + final_W) * 100, 2)) + '%'
        ])
    values.append([
        'Total', total_terrible_M, total_M,
        str(round(total_terrible_M / total_M * 100, 2)) + '%',
        total_terrible_W, total_W,
        str(round(total_terrible_W / total_W * 100, 2)) + '%',
        (total_terrible_M + total_terrible_W),
        (total_M + total_W) - (total_terrible_M + total_terrible_W),
        (total_M + total_W),
        str(
            round((total_terrible_M + total_terrible_W) /
                  (total_M + total_W) * 100, 2)) + '%'
    ])
    return values

def write_excel_xlsx(path, sheet_name, value):
    index = len(value)
    workbook = openpyxl.Workbook()
    sheet = workbook.active
    sheet.title = sheet_name
    for i in range(0, index):
        for j in range(0, len(value[i])):
            sheet.cell(row=i + 1,
                       column=j + 1,
                       value=int(value[i][j])
                       if type(value[i][j]) == int else str(value[i][j]))
    workbook.save(path)
    print("xlsx格式表格写入数据成功！")

def read_excel_xlsx(path, sheet_name):
    workbook = openpyxl.load_workbook(path)
    # sheet = wb.get_sheet_by_name(sheet_name)这种方式已经弃用，不建议使用
    sheet = workbook[sheet_name]
    for row in sheet.rows:
        for cell in row:
            print(cell.value, "\t", end="")
        print()

if __name__ == '__main__':
    book_name_xlsx = 'statics.xlsx'
    sheet_name_xlsx = 'statics'
    write_excel_xlsx(book_name_xlsx, sheet_name_xlsx,
                     statics('F:/human/result/terrible'))
    read_excel_xlsx(book_name_xlsx, sheet_name_xlsx)


# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
import re
import pdb
import argparse
from codecs import open
from tqdm import tqdm
import sys
sys.path.append("../")

def main():
    file_sent   = open('sighan8csc_release1.0/Test/SIGHAN15_CSC_TestInput.txt',
                       'rb', encoding = 'utf-8').readlines()

    file_change = open('sighan8csc_release1.0/Test/SIGHAN15_CSC_TestTruth.txt',
                       'rb', encoding = 'utf-8').readlines()

    for idx, line in enumerate(file_change):
        changes = line[: -1].split(', ')
        num_change = int((len(changes) - 1) / 2 )

        if num_change > 0 and line[: 9] == file_sent[idx][5: 14]:

            for i in range(num_change):

                file_sent[idx] = file_sent[idx][: 15 + int(changes[1:][i * 2])] + \
                                 changes[1:][i * 2 + 1] + \
                                 file_sent[idx][15 + int(changes[1:][i * 2]) + 1:]

        elif line[: 9] != file_sent[idx][5: 14]:
            sys.stderr.write('*************  miss ordered   ************')

    file_output = open('sighan8csc_release1.0/Test/SIGHAN15_CSC_TestTruth_Sentence.txt',
                       'w+', encoding = 'utf-8')

    file_output.write(''.join(file_sent))
    file_output.close()

if __name__ == '__main__':
	main()


from BluenetLib.lib.util.Conversion import Conversion

# voltage is channel 0
# current is channel 1

class AdcChannelPacket:
    packetSize = 6

    def __init__(self, payload, channelIndex):
        if len(payload) < self.packetSize:
            print("ERROR: INVALID PAYLOAD LENGTH", len(payload), payload)
            return

        self.channelIndex = channelIndex

        self.pin    = payload[0]
        self.range  = Conversion.uint8_array_to_uint32(payload[1:1+4])
        self.refPin = payload[5]

    def getDict(self):
        data = {}

        data["pin"] = self.pin
        data["range"] = self.range
        data["refPin"] = self.refPin
        data["channelIndex"] = self.channelIndex

        return data


import os
import shutil
from pathlib import Path

import pytest
from omc3.model.accelerators.accelerator import AcceleratorDefinitionError, AccExcitationMode
from omc3.model.constants import TWISS_AC_DAT, TWISS_ADT_DAT, TWISS_DAT, TWISS_ELEMENTS_DAT
from omc3.model.manager import get_accelerator
from omc3.model.model_creators.lhc_model_creator import LhcBestKnowledgeCreator, LhcModelCreator
from omc3.model_creator import create_instance_and_model

INPUTS = Path(__file__).parent.parent / "inputs"
COMP_MODEL = INPUTS / "models" / "25cm_beam1"
CODEBASE_PATH = Path(__file__).parent.parent.parent / "omc3"
PS_MODEL = CODEBASE_PATH / "model" / "accelerators" / "ps"

@pytest.mark.basic
def test_booster_creation_nominal_driven(tmp_path):
    accel_opt = dict(
        accel="psbooster",
        ring=1,
        nat_tunes=[4.21, 4.27],
        drv_tunes=[0.205, 0.274],
        driven_excitation="acd",
        dpp=0.0,
        energy=0.16,
        modifiers=None,
    )
    accel = create_instance_and_model(
        type="nominal", outputdir=tmp_path, logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["ring"])

@pytest.mark.basic
def test_booster_creation_nominal_free(tmp_path):
    accel_opt = dict(
        accel="psbooster",
        ring=1,
        nat_tunes=[4.21, 4.27],
        dpp=0.0,
        energy=0.16,
        modifiers=None,
    )
    accel = create_instance_and_model(
        type="nominal", outputdir=tmp_path, logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["ring"])

@pytest.mark.basic
def test_ps_creation_nominal_driven_2018(tmp_path):
    accel_opt = dict(
        accel="ps",
        nat_tunes=[6.32, 6.29],
        drv_tunes=[0.325, 0.284],
        driven_excitation="acd",
        dpp=0.0,
        energy=1.4,
        year=2018,
        modifiers=[PS_MODEL / "2018" / "strength" / "PS_LE_LHC_low_chroma.str"],
    )
    accel = create_instance_and_model(
        type="nominal", outputdir=tmp_path, logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["year"])

@pytest.mark.basic
def test_ps_creation_nominal_free_2018(tmp_path):
    accel_opt = dict(
        accel="ps",
        nat_tunes=[6.32, 6.29],
        dpp=0.0,
        energy=1.4,
        year=2018,
        modifiers=[PS_MODEL / "2018" / "strength" / "PS_LE_LHC_low_chroma.str"],
    )
    accel = create_instance_and_model(
        type="nominal", outputdir=tmp_path, logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["year"])

@pytest.mark.basic
def test_ps_creation_nominal_free_2021(tmp_path):
    accel_opt = dict(
        accel="ps",
        nat_tunes=[6.32, 6.29],
        dpp=0.0,
        energy=1.4,
        year=2021,
        modifiers=[PS_MODEL / "2021" / "strength" / "ps_fb_lhc.str"],
    )
    accel = create_instance_and_model(
        type="nominal", outputdir=tmp_path, logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["year"])

@pytest.mark.basic
def test_lhc_creation_nominal_driven(tmp_path):
    accel_opt = dict(
        accel="lhc",
        year="2018",
        ats=True,
        beam=1,
        nat_tunes=[0.31, 0.32],
        drv_tunes=[0.298, 0.335],
        driven_excitation="acd",
        dpp=0.0,
        energy=6.5,
        modifiers=[COMP_MODEL / "opticsfile.24_ctpps2"],
    )
    accel = create_instance_and_model(
        outputdir=tmp_path, type="nominal", logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["beam", "year"])

@pytest.mark.basic
@pytest.mark.parametrize(
    "test_year, uses_ats, uses_run3",
    [("2012", False, False), ("2018", True, False), ("2022", False, True), ("hllhc1.3", False, False)]
)
def test_lhc_creation_use_ats_and_run3_macros(test_year, uses_ats, uses_run3):
    accel_opt = dict(
        accel="lhc",
        year=test_year,
        beam=1,
        nat_tunes=[0.28, 0.31],
        drv_tunes=[0.27, 0.332],
        driven_excitation="acd",
        dpp=0.0,
        energy=6.5,
        modifiers=[COMP_MODEL / "opticsfile.24_ctpps2"],
    )
    accel = get_accelerator(**accel_opt)
    assert accel._uses_ats_knobs() is uses_ats
    assert accel._uses_run3_macros() is uses_run3

@pytest.mark.basic
def test_lhc_creation_nominal_free(tmp_path):
    accel_opt = dict(
        accel="lhc",
        year="2018",
        beam=1,
        nat_tunes=[0.31, 0.32],
        dpp=0.0,
        energy=6.5,
        modifiers=[COMP_MODEL / "opticsfile.24_ctpps2"],
    )
    accel = create_instance_and_model(
        outputdir=tmp_path, type="nominal", logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["beam", "year"])

@pytest.mark.basic
def test_lhc_creation_best_knowledge(tmp_path):
    (tmp_path / LhcBestKnowledgeCreator.EXTRACTED_MQTS_FILENAME).write_text("\n")
    (tmp_path / LhcBestKnowledgeCreator.CORRECTIONS_FILENAME).write_text("\n")
    accel_opt = dict(
        accel="lhc",
        year="2018",
        ats=True,
        beam=1,
        nat_tunes=[0.31, 0.32],
        dpp=0.0,
        energy=6.5,
        modifiers=[COMP_MODEL / "opticsfile.24_ctpps2"],
    )
    accel = create_instance_and_model(
        outputdir=tmp_path, type="best_knowledge", logfile=tmp_path / "madx_log.txt", **accel_opt
    )

@pytest.mark.basic
def test_lhc_creation_relative_modifier_path(tmp_path):
    accel_opt = dict(
        accel="lhc",
        year="2018",
        ats=True,
        beam=1,
        nat_tunes=[0.31, 0.32],
        dpp=0.0,
        energy=6.5,
        modifiers=[Path("opticsfile.24_ctpps2")],
    )
    shutil.copy(COMP_MODEL / "opticsfile.24_ctpps2", tmp_path / "opticsfile.24_ctpps2")

    accel = create_instance_and_model(
        outputdir=tmp_path, type="nominal", logfile=tmp_path / "madx_log.txt", **accel_opt
    )
    check_accel_from_dir_vs_options(tmp_path, accel_opt, accel, required_keys=["beam", "year"])

@pytest.mark.basic
def test_lhc_creation_modifier_nonexistent(tmp_path):
    accel_opt = dict(
        accel="lhc",
        year="2018",
        ats=True,
        beam=1,
        nat_tunes=[0.31, 0.32],
        dpp=0.0,
        energy=6.5,
        modifiers=[COMP_MODEL / "opticsfile.non_existent"],
    )
    with pytest.raises(AcceleratorDefinitionError) as creation_error:
        create_instance_and_model(
            outputdir=tmp_path, type="nominal", logfile=tmp_path / "madx_log.txt", **accel_opt
        )
    assert "opticsfile.non_existent" in str(creation_error.value)

@pytest.mark.basic
@pytest.mark.timeout(60)  # madx might get stuck (seen on macos)
def test_lhc_creation_relative_modeldir_path(request, tmp_path):
    os.chdir(tmp_path)  # switch cwd to tmp_path
    model_dir_relpath = Path("test_model")
    model_dir_relpath.mkdir()

    optics_file_relpath = Path("opticsfile.24_ctpps2")
    shutil.copy(COMP_MODEL / optics_file_relpath, model_dir_relpath / optics_file_relpath)

    accel_opt = dict(
        accel="lhc",
        year="2018",
        ats=True,
        beam=1,
        nat_tunes=[0.31, 0.32],
        dpp=0.0,
        energy=6.5,
        modifiers=[optics_file_relpath],
    )

    # sometimes create_instance_and_model seems to run but does not create twiss-files ...
    accel = create_instance_and_model(
        outputdir=model_dir_relpath, type="nominal", logfile=tmp_path / "madx_log.txt", **accel_opt
    )

    # ... which is then caught here:
    check_accel_from_dir_vs_options(
        model_dir_relpath, accel_opt, accel, required_keys=["beam", "year"]
    )
    os.chdir(request.config.invocation_dir)  # return to original cwd

@pytest.mark.basic
def test_lhc_creation_nominal_driven_check_output(model_25cm_beam1):
    accel = get_accelerator(**model_25cm_beam1)
    LhcModelCreator.check_run_output(accel)

    for dat_file in (TWISS_AC_DAT, TWISS_DAT, TWISS_ELEMENTS_DAT, TWISS_ADT_DAT):
        file_path: Path = accel.model_dir / dat_file
        file_path_moved: Path = file_path.with_suffix(f"{file_path.suffix}0")
        if dat_file == TWISS_ADT_DAT:
            accel.excitation = AccExcitationMode.ADT  # Test ACD before
        else:
            shutil.move(file_path, file_path_moved)

        # Run test
        with pytest.raises(FileNotFoundError) as creation_error:
            LhcModelCreator.check_run_output(accel)
        assert str(dat_file) in str(creation_error.value)

        if file_path_moved.exists():
            shutil.move(file_path_moved, file_path)

# Helper -----------------------------------------------------------------------

def check_accel_from_dir_vs_options(model_dir, accel_options, accel_from_opt, required_keys):
    # creation via model_from_dir tests that all files are in place:
    accel_from_dir = get_accelerator(
        accel=accel_options["accel"],
        model_dir=model_dir,
        **{k: accel_options[k] for k in required_keys},
    )

    _check_arrays(accel_from_opt.nat_tunes, accel_from_dir.nat_tunes, eps=1e-4, tunes=True)
    _check_arrays(accel_from_opt.drv_tunes, accel_from_dir.drv_tunes, eps=1e-4, tunes=True)
    _check_arrays(accel_from_opt.modifiers, accel_from_dir.modifiers)
    assert accel_from_opt.excitation == accel_from_dir.excitation
    assert accel_from_opt.model_dir == accel_from_dir.model_dir

    # TODO: Energy not set in model ? (jdilly, 2021)
    # assert abs(accel_from_opt.energy - accel_from_dir.energy) < 1e-2

def _check_arrays(a_array, b_array, eps=None, tunes=False):
    if a_array is None:
        a_array = []

    if b_array is None:
        b_array = []

    if len(a_array) != len(b_array):
        raise AssertionError("Not the same amounts given.")

    for a, b in zip(a_array, b_array):
        if eps is None:
            assert a == b
        elif tunes:
            assert abs((a % 1) - (b % 1)) <= eps
        else:
            assert abs(a - b) <= eps


import collections.abc
from abc import ABC, abstractmethod
from collections import defaultdict
from typing import Collection, Mapping, NamedTuple, Optional, Union, cast

import dagster._check as check
from dagster._annotations import PublicAttr, experimental, public
from dagster._core.definitions.multi_dimensional_partitions import (
    MultiPartitionKey,
    MultiPartitionsDefinition,
)
from dagster._core.definitions.partition import (
    PartitionsDefinition,
    PartitionsSubset,
    StaticPartitionsDefinition,
)
from dagster._core.definitions.partition_key_range import PartitionKeyRange
from dagster._serdes import whitelist_for_serdes
from dagster._utils.cached_method import cached_method

class PartitionMapping(ABC):
    """Defines a correspondence between the partitions in an asset and the partitions in an asset
    that it depends on.

    Overriding PartitionMapping outside of Dagster is not supported. The abstract methods of this
    class may change at any time.
    """

    @public
    @abstractmethod
    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        """Returns the range of partition keys in the upstream asset that include data necessary
        to compute the contents of the given partition key range in the downstream asset.

        Args:
            downstream_partition_key_range (PartitionKeyRange): The range of partition keys in the
                downstream asset.
            downstream_partitions_def (PartitionsDefinition): The partitions definition for the
                downstream asset.
            upstream_partitions_def (PartitionsDefinition): The partitions definition for the
                upstream asset.
        """

    @public
    @abstractmethod
    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        """Returns the range of partition keys in the downstream asset that use the data in the given
        partition key range of the upstream asset.

        Args:
            upstream_partition_key_range (PartitionKeyRange): The range of partition keys in the
                upstream asset.
            downstream_partitions_def (PartitionsDefinition): The partitions definition for the
                downstream asset.
            upstream_partitions_def (PartitionsDefinition): The partitions definition for the
                upstream asset.
        """

    @public
    def get_upstream_partitions_for_partitions(
        self,
        downstream_partitions_subset: Optional[PartitionsSubset],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        """
        Returns the subset of partition keys in the upstream asset that include data necessary
        to compute the contents of the given partition key subset in the downstream asset.

        Args:
            downstream_partitions_subset (Optional[PartitionsSubset]):
                The subset of partition keys in the downstream asset.
            upstream_partitions_def (PartitionsDefinition): The partitions definition for the
                upstream asset.
        """
        upstream_key_ranges = []
        if downstream_partitions_subset is None:
            upstream_key_ranges.append(
                self.get_upstream_partitions_for_partition_range(
                    None, None, upstream_partitions_def
                )
            )
        else:
            for key_range in downstream_partitions_subset.get_partition_key_ranges():
                upstream_key_ranges.append(
                    self.get_upstream_partitions_for_partition_range(
                        key_range,
                        downstream_partitions_subset.partitions_def,
                        upstream_partitions_def,
                    )
                )

        return upstream_partitions_def.empty_subset().with_partition_keys(
            pk
            for upstream_key_range in upstream_key_ranges
            for pk in upstream_partitions_def.get_partition_keys_in_range(upstream_key_range)
        )

    @public
    def get_downstream_partitions_for_partitions(
        self,
        upstream_partitions_subset: PartitionsSubset,
        downstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        """
        Returns the subset of partition keys in the downstream asset that use the data in the given
        partition key subset of the upstream asset.

        Args:
            upstream_partitions_subset (Union[PartitionKeyRange, PartitionsSubset]): The
                subset of partition keys in the upstream asset.
            downstream_partitions_def (PartitionsDefinition): The partitions definition for the
                downstream asset.
        """
        downstream_key_ranges = []
        for key_range in upstream_partitions_subset.get_partition_key_ranges():
            downstream_key_ranges.append(
                self.get_downstream_partitions_for_partition_range(
                    key_range,
                    downstream_partitions_def,
                    upstream_partitions_subset.partitions_def,
                )
            )

        return downstream_partitions_def.empty_subset().with_partition_keys(
            pk
            for upstream_key_range in downstream_key_ranges
            for pk in downstream_partitions_def.get_partition_keys_in_range(upstream_key_range)
        )

@whitelist_for_serdes
class IdentityPartitionMapping(PartitionMapping, NamedTuple("_IdentityPartitionMapping", [])):
    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        if downstream_partitions_def is None or downstream_partition_key_range is None:
            check.failed("downstream asset is not partitioned")

        return downstream_partition_key_range

    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        return upstream_partition_key_range

@whitelist_for_serdes
class AllPartitionMapping(PartitionMapping, NamedTuple("_AllPartitionMapping", [])):
    def get_upstream_partitions_for_partitions(
        self,
        downstream_partitions_subset: Optional[PartitionsSubset],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        first = upstream_partitions_def.get_first_partition_key()
        last = upstream_partitions_def.get_last_partition_key()

        empty_subset = upstream_partitions_def.empty_subset()
        if first is not None and last is not None:
            return empty_subset.with_partition_key_range(PartitionKeyRange(first, last))
        else:
            return empty_subset

    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

@whitelist_for_serdes
class LastPartitionMapping(PartitionMapping, NamedTuple("_LastPartitionMapping", [])):
    def get_upstream_partitions_for_partitions(
        self,
        downstream_partitions_subset: Optional[PartitionsSubset],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        last = upstream_partitions_def.get_last_partition_key()

        empty_subset = upstream_partitions_def.empty_subset()
        if last is not None:
            return empty_subset.with_partition_keys([last])
        else:
            return empty_subset

    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

@experimental
class SingleDimensionDependencyMapping(PartitionMapping):
    """
    Defines a correspondence between an upstream single-dimensional partitions definition
    and a downstream MultiPartitionsDefinition. The upstream partitions definition must be
    a dimension of the downstream MultiPartitionsDefinition.

    For an upstream partition key X, this partition mapping assumes that any downstream
    multi-partition key with X in the selected dimension is a downstream dependency.

    Args:
        partition_dimension_name (str): The name of the partition dimension in the downstream
            MultiPartitionsDefinition that matches the upstream partitions definition.
    """

    def __init__(self, partition_dimension_name: str):
        self.partition_dimension_name = partition_dimension_name

    def _check_upstream_partitions_def_equals_selected_dimension(
        self,
        upstream_partitions_def: PartitionsDefinition,
        downstream_partitions_def: MultiPartitionsDefinition,
    ) -> None:
        matching_dimensions = [
            partitions_def
            for partitions_def in downstream_partitions_def.partitions_defs
            if partitions_def.name == self.partition_dimension_name
        ]
        if len(matching_dimensions) != 1:
            check.failed(f"Partition dimension '{self.partition_dimension_name}' not found")
        matching_dimension_def = next(iter(matching_dimensions))

        if upstream_partitions_def != matching_dimension_def.partitions_def:
            check.failed(
                "The upstream partitions definition does not have the same partitions definition "
                f"as dimension {matching_dimension_def.name}"
            )

    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

    def get_upstream_partitions_for_partitions(
        self,
        downstream_partitions_subset: Optional[PartitionsSubset],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        if downstream_partitions_subset is None or not isinstance(
            downstream_partitions_subset.partitions_def, MultiPartitionsDefinition
        ):
            check.failed("downstream asset is not partitioned")

        self._check_upstream_partitions_def_equals_selected_dimension(
            upstream_partitions_def, downstream_partitions_subset.partitions_def
        )
        upstream_partitions = set()
        for partition_key in downstream_partitions_subset.get_partition_keys():
            if not isinstance(partition_key, MultiPartitionKey):
                check.failed(
                    "Partition keys in downstream partition key subset must be MultiPartitionKeys"
                )
            upstream_partitions.add(partition_key.keys_by_dimension[self.partition_dimension_name])
        return upstream_partitions_def.empty_subset().with_partition_keys(upstream_partitions)

    def get_downstream_partitions_for_partitions(
        self,
        upstream_partitions_subset: PartitionsSubset,
        downstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        if downstream_partitions_def is None or not isinstance(
            downstream_partitions_def, MultiPartitionsDefinition
        ):
            check.failed("downstream asset is not multi-partitioned")

        self._check_upstream_partitions_def_equals_selected_dimension(
            upstream_partitions_subset.partitions_def, downstream_partitions_def
        )

        upstream_keys = list(upstream_partitions_subset.get_partition_keys())

        matching_keys = []
        for key in downstream_partitions_def.get_partition_keys():
            key = cast(MultiPartitionKey, key)
            if key.keys_by_dimension[self.partition_dimension_name] in upstream_keys:
                matching_keys.append(key)

        return downstream_partitions_def.empty_subset().with_partition_keys(set(matching_keys))

@whitelist_for_serdes
class StaticPartitionMapping(
    PartitionMapping,
    NamedTuple(
        "_StaticPartitionMapping",
        [
            (
                "downstream_partition_keys_by_upstream_partition_key",
                PublicAttr[Mapping[str, Union[str, Collection[str]]]],
            )
        ],
    ),
):
    """
    Define an explicit correspondence between two StaticPartitionsDefinitions.

    Args:
        downstream_partition_keys_by_upstream_partition_key (Dict[str, str | Collection[str]]):
          The single or multi-valued correspondence from upstream keys to downstream keys.
    """

    def __init__(
        self,
        downstream_partition_keys_by_upstream_partition_key: Mapping[
            str, Union[str, Collection[str]]
        ],
    ):
        check.mapping_param(
            downstream_partition_keys_by_upstream_partition_key,
            "downstream_partition_keys_by_upstream_partition_key",
            key_type=str,
            value_type=(str, collections.abc.Collection),
        )

        # cache forward and reverse mappings
        self._mapping = defaultdict(set)
        for (
            upstream_key,
            downstream_keys,
        ) in downstream_partition_keys_by_upstream_partition_key.items():
            self._mapping[upstream_key] = (
                {downstream_keys} if isinstance(downstream_keys, str) else set(downstream_keys)
            )

        self._inverse_mapping = defaultdict(set)
        for upstream_key, downstream_keys in self._mapping.items():
            for downstream_key in downstream_keys:
                self._inverse_mapping[downstream_key].add(upstream_key)

    @cached_method
    def _check_upstream(self, *, upstream_partitions_def: PartitionsDefinition):
        """
        validate that the mapping from upstream to downstream is only defined on upstream keys
        """
        check.inst(
            upstream_partitions_def,
            StaticPartitionsDefinition,
            "StaticPartitionMapping can only be defined between two StaticPartitionsDefinitions",
        )
        upstream_keys = upstream_partitions_def.get_partition_keys()
        extra_keys = set(self._mapping.keys()).difference(upstream_keys)
        if extra_keys:
            raise ValueError(
                f"mapping source partitions not in the upstream partitions definition: {extra_keys}"
            )

    @cached_method
    def _check_downstream(self, *, downstream_partitions_def: PartitionsDefinition):
        """
        validate that the mapping from upstream to downstream only maps to downstream keys
        """
        check.inst(
            downstream_partitions_def,
            StaticPartitionsDefinition,
            "StaticPartitionMapping can only be defined between two StaticPartitionsDefinitions",
        )
        downstream_keys = downstream_partitions_def.get_partition_keys()
        extra_keys = set(self._inverse_mapping.keys()).difference(downstream_keys)
        if extra_keys:
            raise ValueError(
                "mapping target partitions not in the downstream partitions definition:"
                f" {extra_keys}"
            )

    def get_downstream_partitions_for_partitions(
        self,
        upstream_partitions_subset: PartitionsSubset,
        downstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        self._check_downstream(downstream_partitions_def=downstream_partitions_def)

        downstream_subset = downstream_partitions_def.empty_subset()
        downstream_keys = set()
        for key in upstream_partitions_subset.get_partition_keys():
            downstream_keys.update(self._mapping[key])
        return downstream_subset.with_partition_keys(downstream_keys)

    def get_upstream_partitions_for_partitions(
        self,
        downstream_partitions_subset: Optional[PartitionsSubset],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionsSubset:
        self._check_upstream(upstream_partitions_def=upstream_partitions_def)

        upstream_subset = upstream_partitions_def.empty_subset()
        if downstream_partitions_subset is None:
            return upstream_subset

        upstream_keys = set()
        for key in downstream_partitions_subset.get_partition_keys():
            upstream_keys.update(self._inverse_mapping[key])

        return upstream_subset.with_partition_keys(upstream_keys)

    def get_upstream_partitions_for_partition_range(
        self,
        downstream_partition_key_range: Optional[PartitionKeyRange],
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

    def get_downstream_partitions_for_partition_range(
        self,
        upstream_partition_key_range: PartitionKeyRange,
        downstream_partitions_def: Optional[PartitionsDefinition],
        upstream_partitions_def: PartitionsDefinition,
    ) -> PartitionKeyRange:
        raise NotImplementedError()

def infer_partition_mapping(
    partition_mapping: Optional[PartitionMapping], partitions_def: Optional[PartitionsDefinition]
) -> PartitionMapping:
    if partition_mapping is not None:
        return partition_mapping
    elif partitions_def is not None:
        return partitions_def.get_default_partition_mapping()
    else:
        return AllPartitionMapping()

def get_builtin_partition_mapping_types():
    from dagster._core.definitions.time_window_partition_mapping import TimeWindowPartitionMapping

    return (
        AllPartitionMapping,
        IdentityPartitionMapping,
        LastPartitionMapping,
        StaticPartitionMapping,
        TimeWindowPartitionMapping,
    )


def test (s):
    try:
        float(s)
        return True
    except ValueError:
        return False
while True:
        a = input("Ведіть перше число")
        b = input("Ведіть друге число")
        c = input("Ведіть третє число")
        if test(a) == False or test(b) == False or test(c) == False:
            print ("Ведіть ,будь ласка, число")
        else:
            a = float(a)
            b = float(b)
            c = float(c)
            result = a+b+c
            print(result)


import sys
import os

PROJECTDIR = os.getcwd()
GAN_DIR = os.path.join(PROJECTDIR, "ganspace")
sys.path.insert(0, GAN_DIR)

import numpy as np
import pandas as pd

import ipywidgets as widgets

from jupGUI_utils.utils import verbose_info, widget_image_to_bytes

VERBOSE = False
VERBOSE_ENDL = "\t   "

class GUIses(object):
    def __init__(self,
                 ganpreffinder):
        """
        :param ganpreffinder: GANPrefFinder class instance
        """
        self.optimize_Theta = False

        self.GANpf = ganpreffinder

        WIDGET_CONTINIOUS_UPDATE = self.GANpf.USING_CUDA
        WIDGET_WIDTH = 300

        self.strength = 0
        self.w = self.GANpf.get_init_W()
        init_img = self.GANpf.get_init_img()

        self.X, self.Xi = self.GANpf.get_next_query()

        self.widget_image = widgets.Image(format='png', width=WIDGET_WIDTH,
                                          value=self.to_bytes(init_img))

        self.widget_pref_image = widgets.Image(format='png', width=WIDGET_WIDTH,
                                               value=self.to_bytes(init_img))

        self.widget_image_init = widgets.Image(format='png', width=WIDGET_WIDTH,
                                               value=self.to_bytes(init_img))

        self.widget_strength_slider = widgets.IntSlider(min=self.GANpf.left_bound,
                                                        max=self.GANpf.right_bound,
                                                        value=self.strength,
                                                        continuous_update=WIDGET_CONTINIOUS_UPDATE,
                                                        description='strength:')

        self.widget_strength_text = widgets.IntText(description="strength:",
                                                    continuous_update=WIDGET_CONTINIOUS_UPDATE)

        self.widget_button_incs = widgets.Button(description="slider +10 (both sides)",
                                                 buttin_style="info",
                                                 layout=widgets.Layout(width="150px"))

        self.widget_button_decs = widgets.Button(description="slider -10 (both sides)",
                                                 buttin_style="info",
                                                 layout=widgets.Layout(width="150px"))

        self.widget_button_nq = widgets.Button(description="next query",
                                               buttin_style="info",
                                               layout=widgets.Layout(width="300px"))

        self.widget_chb_adaptive_init = widgets.Checkbox(
            value=self.GANpf.ADAPTIVE_INITIALIZATION,
            description='adaptive initialization', disabled=False, indent=False)

    def run(self):
        self.widget_strength_text.observe(self.change_strength, 'value')
        self.widget_button_nq.on_click(self.next_query)
        self.widget_button_incs.on_click(self.increase_slider_range)
        self.widget_button_decs.on_click(self.decrease_slider_range)
        self.widget_chb_adaptive_init.observe(self.switch_adaptive_init)
        # self.widget_chb_choose_comp.observe(self.switch_Xi_strategy)
        # self.widget_comp_slider.observe(self.change_component)

        widgets.link((self.widget_strength_slider, 'value'), (self.widget_strength_text, 'value'))

        manage_panel = widgets.VBox([
            widgets.Label(value='Control Panel:'),
            #widgets.HBox([self.widget_button_decs, self.widget_button_incs]),
            self.widget_strength_slider,
            self.widget_strength_text,
            self.widget_chb_adaptive_init,
            self.widget_button_nq
        ])

        image_panel = widgets.VBox([widgets.Label(value='Image:'), self.widget_image])

        pref_image_panel = widgets.VBox([widgets.Label(value='Preferred Image:'), self.widget_pref_image])

        init_image_panel = widgets.VBox([widgets.Label(value='Initial Image:'), self.widget_image_init])

        run_wid = widgets.VBox([
            widgets.HBox([image_panel, pref_image_panel, init_image_panel]),
            manage_panel
        ])

        return run_wid

    # === easy switching components ===

    def switch_adaptive_init(self, _):
        self.GANpf.switch_adaptive_initialization()
        if not self.GANpf.ADAPTIVE_INITIALIZATION:
            self.optimize_Theta = True

    def increase_slider_range(self, _):
        self.widget_strength_slider.min -= 10
        self.widget_strength_slider.max += 10

    def decrease_slider_range(self, _):
        self.widget_strength_slider.min += 10
        self.widget_strength_slider.max -= 10

    # === main functions ===

    def change_strength(self, strength):
        self._update_strength_value(strength.new)
        self._update_image()

    def next_query(self, _):
        self._updateGP()
        self._update_pref_image()
        # if self.optimize_Theta:
        #     print("+++ OPTIMIZING Theta for GP")
        #     self.ganpfinder.optimizeGP()
        #     self.optimize_Theta = False
        #     print("+++ FINISHED Theta OPTIMIZATION")
        self._update_X()
        self._next_query()
        self._update_strength_value(0)
        self._update_image()
        self._update_strength_slider_value()

    # === help functions ===

    @verbose_info(verbose=VERBOSE, msg="+ Updating GP", verbose_endl=VERBOSE_ENDL)
    def _updateGP(self):
        self.GANpf.updateGP(self.X, self.Xi, self.strength)

    @verbose_info(verbose=VERBOSE, msg="+ Getting next query", verbose_endl=VERBOSE_ENDL)
    def _next_query(self):
        self.X, self.Xi = self.GANpf.get_next_query()

    def _update_X(self):
        self.GANpf.update_adaptive_query(self.X, self.Xi, self.strength)

    @verbose_info(verbose=VERBOSE, msg="+ Updating image", verbose_endl=VERBOSE_ENDL)
    def _update_image(self):
        prefVec = self.GANpf.calculate_pref_vector(self.X, self.Xi, self.strength)
        layers_range = self.GANpf.get_comp_layers_range(self.Xi)
        img = self.GANpf.update_image(prefVec=prefVec, layers_range=layers_range)
        self.widget_image.value = self.to_bytes(img)

    @verbose_info(verbose=VERBOSE, msg="+ Updating preferred image", verbose_endl=VERBOSE_ENDL)
    def _update_pref_image(self):
        X_star = self.GANpf.get_last_X_star_scaled()
        img = self.GANpf.update_image(prefVec=X_star)
        self.widget_pref_image.value = self.to_bytes(img)

    def _update_strength_slider_value(self):
        self.widget_strength_slider.value = self.strength

    @verbose_info(verbose=VERBOSE, msg="+ Updating strength parameter", verbose_endl=VERBOSE_ENDL)
    def _update_strength_value(self, new_value):
        self.strength = new_value

    # ===============================

    @staticmethod
    def to_bytes(img):
        return widget_image_to_bytes(img)

    # === next query ===

    # def _move_user_Xi(self):
    #     if self.ganpfinder.AQ.custom_Xi_strategy:
    #         if self.widget_comp_slider.value < self.widget_comp_slider.max:
    #             self.widget_comp_slider.value += 1
    #         else:
    #             self.widget_comp_slider.value += 1
    #
    # @verbose_info(verbose=VERBOSE, msg="+ Updating GUI parameters", verbose_endl=VERBOSE_ENDL)
    # def _updateGUI(self):
    #     self.strength = 0
    #     self._move_user_Xi()
    #     if self.start_with_init_img:
    #         img = self.ganpfinder.get_init_img()
    #     else:
    #         prefVec = self.ganpfinder.calculate_pref_vector(self.X, self.Xi, self.strength)
    #         img = self.ganpfinder.update_image(prefVec=prefVec)
    #     self.widget_image.value = self.to_bytes(img)
    #
    # # ==================
    #
    # # === custom change component ===
    #
    # def change_component(self, comp_index):
    #     comp_index_value = comp_index.new
    #     if type(comp_index_value) is int:
    #         self._Xi_from_user(comp_index_value)
    #         if self.ganpfinder.AQ.custom_Xi_strategy:
    #             self._update_image()
    #
    # @verbose_info(verbose=VERBOSE, msg="+ Updating component switch parameter", verbose_endl=VERBOSE_ENDL)
    # def _Xi_from_user(self, comp_index_value):
    #     self.ganpfinder.Xi_from_user(comp_index_value)
    #     self.Xi = self.ganpfinder.AQ.get_Xi()
    #
    # @verbose_info(verbose=VERBOSE, msg="+ Updating GUI parameters", verbose_endl=VERBOSE_ENDL)
    # def _update_image(self):
    #     img = self._modify_by_strength(self.strength)
    #     self.widget_image.value = self.to_bytes(img)


#-*- coding=utf-8 -*-
#-----author:GYl-----
#-----date:2018/12/13-----

import socket
import threading

bind_ip = "0.0.0.0"
bind_port= 8888

server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)

server.bind((bind_ip,bind_port))

server.listen(5)

print ("[*] listing  on %s:%d"%(bind_ip,bind_port))

def handle_client(client_socket):
    request = client_socket.recv(1024)
    print ('[*] received:%s'%request)

    #返回一个数据包
    client_socket.send(bytes("ack",encoding='utf-8'))
    client_socket.close()

while True:
    client,addr = server.accept()
    print ("[*] Accept connection form:%s:%d"%(addr[0],addr[1]))

    #挂起客户端线程，处理传入的数据
    client_handler=threading.Thread(target=handle_client,args=(client,))
    client_handler.start()


""" Module with user model schemas. """
import re
from marshmallow import (ValidationError, fields, post_load)

from api.models.user import User
from ..validators.name_validator import name_validator
from ..validators.string_length_validators import string_length_60_validator
from ..messages.error_messages import serialization_errors
from .base_schemas import BaseSchema

email_regex = re.compile(
    r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")

def email_exists_validator(data):
    """
    Checks if given email has correct email syntax & not
    registered already. Raises a ValidationError otherwise
    """

    if not re.match(email_regex, data):
        raise ValidationError(
            serialization_errors['email_syntax'].format(data))
    elif User.query.filter_by(email=data).first():
        raise ValidationError(
            serialization_errors['email_exists'].format(data))
    else:
        raise ValidationError(serialization_errors['email_length'])

class UserSchema(BaseSchema):
    """ User model schema. """

    name = fields.String(required=True,
                         validate=(string_length_60_validator,
                                   name_validator),
                         error_messages={
                             'required':
                             serialization_errors['field_required']})
    email = fields.String(required=True,
                          validate=email_exists_validator,
                          error_messages={
                              'required':
                              serialization_errors['field_required']})
    image_url = fields.String()

    @post_load
    def create_user(self, data):
        """ Return user object after successful loading of data"""

        return User(**data)


# __author__: Stanley
# date: 2018/10/23

class Foo:
    __v = None

    @classmethod
    def get_instance(cls):
        if cls.__v:
            return cls.__v
        else:
            cls.__v = Foo()
            return cls.__v

obj = Foo.get_instance()
print(obj)

# class Foo:
#     __v = None
#
#     def get_instance(self):
#         if self.__v:
#             return self.__v
#         else:
#             self.__v = Foo()
#             return self.__v
#
#
# obj = Foo()
# print(obj.get_instance())

## 封装
# class F1:
#     def __init__(self):
#         self.name = 123
#
#
# class F2:
#     def __init__(self, a):
#         self.ff = a
#
#
# class F3:
#     def __init__(self, b):
#         self.dd = b
#
#
# f1 = F1()
# f2 = F2(f1)
# f3 = F3(f2)
#
# print(f1.name)
# print(f2.ff.name)
# print(f3.dd.ff.name)


# tuple - это упорядоченная неизменяемая последовательность элементов.
# На русском правильно говорить - кортеж.

# обычно tuple объявляют так:
some_tuple1 = ('some str', 131, 46.74)

# Но можно это сделать без скобок. Показателем того что это tuple являются ЗАПЯТЫЕ!
some_tuple2 = 'x', 131, 46

z = 1
y = 2

z, y = y, z

# А еще tuple используют для множественного присваивания.
# Пример:
age, name = 23, 'SomeName1'
# Здесь tuple c левой стороны РАСПАКОВЫВАЕТСЯ в tuple с правой.
# При этом, Вы можете как распокавать уже существующий tuple
age1 = 23
name1 = 'SomeName'
age_and_name = age1, name1

new_age2, new_name2 = age_and_name

# При этом если например, вам нужно получить первый элемент кортежа и последний, а другие вам не нужны,
# вы можете сделать так:
some_long_tuple: tuple = ('first elem', 2, 3, 4, 5, 6, 'last elem')
first_elem, *other_elements, last_elem = some_long_tuple
# В данном случае вы получите:
# first_elem: str = 'first elem'
# other_elements: tuple = (2, 3, 4, 5, 6)
# last_elem: str = 'last elem'

# еще пример:
first_elem1, second_elem1, *other_elements1 = some_long_tuple
# В данном случае вы получите:
# first_elem: str = 'first elem'
# second_elem1: int = 2
# last_elem: tuple = (2, 3, 4, 5, 6, 'last elem')

# Этот несложный трюк иногда очень облегчает работу с последовательностями

Additional files/Lesson 4/3. tuple, множественное присванивание.py


def happy_ticket(num):
    num_list = [i for i in str(num)]
    if sum([int(i) for i in num_list[:3]]) == sum([int(i) for i in num_list[3:]]):
        print("Счастливый")
    else:
        print("Обычный")

if __name__ == "__main__":
    # Write your solution here
    happy_num = int(input('Введите число: '))
    print(happy_ticket(happy_num))

Занятие4/Домашнее_задание/task4/main.py


import torch
import torch.nn as nn
import torch.utils.data as data_utils
from torch.autograd import Variable

import numpy as np
import matplotlib.pylab as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
#from model_nn import *
#from model_dnn import *
from model_cnn import *
#from model_rnn import *
from dataload import dataload
from label_filter import label_filter
from similarity import similarity

from parameter import *

# data dir
data_dir=data_dir+'test/'
# Hyper Parameters
batch_size = 1

train_set=np.empty
label_set=np.empty

test_set, label_set, data_dsp_dim, label_dsp_dim = dataload(test_size,data_dir,data_dim,model_dim,data_dsp_blk,label_dsp_blk)

num_classes = label_dsp_dim[0]*label_dsp_dim[1]
input_size=data_dsp_dim[0]*data_dsp_dim[1]

# data loader
test = data_utils.TensorDataset(torch.from_numpy(test_set), torch.from_numpy(label_set))
test_loader = data_utils.DataLoader(test, batch_size=batch_size, shuffle=False)

# model loader
#net = Net(input_size, hidden_size1, num_classes)
#net = Net(input_size, hidden_size1, hidden_size2, hidden_size3, hidden_size4, num_classes)
net = Net(input_size, data_dsp_dim, num_classes)  # CNN
#net = Net(data_dsp_dim[1],hidden_size,num_layers, num_classes)  # RNN

net.load_state_dict(torch.load(model_adr))

# Test the Model
nx=model_dim[1]
nz=model_dim[0]
correct = 0
total = 0
thr=0.03
for images, labels in test_loader:
   # images = Variable(images.view(-1, data_dsp_dim[0]*data_dsp_dim[1]))
    images = Variable(images.view(batch_size,1,data_dsp_dim[0],data_dsp_dim[1]))   # CNN
    #images = Variable(images.view(-1,data_dsp_dim[0],data_dsp_dim[1]))   # RNN
    outputs = net(images)
    total=total+1
    # plot
    PD=outputs.data.numpy().reshape(label_dsp_dim[0],label_dsp_dim[1])
    PD=label_filter(PD,thr)
    GT=labels.numpy().reshape(label_dsp_dim[0],label_dsp_dim[1])
#    GT=label_filter(GT,thr)
#    fig, (ax1,ax2,ax3) = plt.subplots(1,3)
    fig, (ax1,ax2) = plt.subplots(2,1)
    im1=ax1.imshow(GT,extent=[0,nx*dh,0,nz*dh])#,aspect='auto',cmap='bwr')
    divider = make_axes_locatable(ax1)
    cax1 = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im1,ax=ax1,cax=cax1).set_label('Probability')
    ax1.set_xlabel('Position (m)')
    ax1.set_ylabel('Depth (m)')
    ax1.set_title('Ground truth',y=1.28)
    ax1.invert_yaxis()
    ax1.xaxis.tick_top()
    ax1.xaxis.set_label_position('top')
#    prd=PD/np.amax(PD)
    im2=ax2.imshow(PD,extent=[0,nx*dh,0,nz*dh])#,aspect='auto',cmap='bwr')
    divider = make_axes_locatable(ax2)
    cax2 = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im2,ax=ax2, cax=cax2).set_label('Probability')
#    ax2.set_xlabel('Position (m)')
    ax2.set_ylabel('Depth (m)')
    ax2.set_title('Prediction',y=1.18)
    ax2.invert_yaxis()
    ax2.xaxis.tick_top()

#    im3=ax3.imshow(images.data.numpy().reshape(data_dsp_dim[0],data_dsp_dim[1]))
#    divider = make_axes_locatable(ax3)
#    cax3 = divider.append_axes("right", size="5%", pad=0.05)
#    plt.colorbar(im3,ax=ax3, cax=cax3)

    plt.tight_layout(pad=2)
    figname='figure'+str(total)
    #plt.savefig(figname, transparent=True)
    plt.show()


class Node(object):
	def __init__(self, data, nextNode=None):
		self.data = data
		self.nextNode = nextNode

class LinkedList(object):
	def __init__(self, sequence=[]):
		current = None
		for i in sequence:
			if not current:
				self.head = Node(i)
				current = self.head
			else:
				current.nextNode = Node(i)
				current = current.nextNode

	def find(self, target):
		current = self.head
		while current:
			if current.data == target:
				return current
			current = current.nextNode
		return None

ll = LinkedList([1])
assert ll.find(1).data == 1

ll = LinkedList([1, 8, 9, 10, 11, 12])
assert ll.find(11).nextNode.data == 12