import sys

# Enthought library imports
from pyface.qt import QtCore, QtGui

# Local imports
from tvtk.util.gradient_editor import (
    ColorControlPoint, ChannelBase, FunctionControl, GradientEditorWidget
)

##########################################################################
# `QGradientControl` class.
##########################################################################
class QGradientControl(QtGui.QWidget):
    """Widget which displays the gradient represented by an GradientTable
    object (and does nothing beyond that)"""
    def __init__(self, parent=None, gradient_table=None, width=100, height=100):
        """master: panel in which to place the control. GradientTable is the
        Table to which to attach."""
        super(QGradientControl, self).__init__(parent=parent)
        self.resize(width, height)
        self.setAttribute(QtCore.Qt.WA_OpaquePaintEvent, True)

        self.width = width
        self.height = height
        self.gradient_table = gradient_table
        assert gradient_table.size == width
        self.setMinimumSize(100, 50)
        # currently only able to use gradient tables in the same size as the
        # canvas width

    def paintEvent(self, event):
        """Paint handler."""
        super(QGradientControl, self).paintEvent(event)

        painter = QtGui.QPainter(self)
        brush = QtGui.QBrush(QtGui.QColor(0, 0, 0) )
        painter.setBrush(brush)
        painter.setBackgroundMode(QtCore.Qt.OpaqueMode)
        sz = self.size()
        width, height = sz.width(), sz.height()

        xform = self.gradient_table.scaling_function
        start_y = 0
        end_y = height
        if xform:
            # if a scaling transformation is provided, paint the original
            # gradient under the scaled gradient.
            start_y = height/2

        # paint the original gradient as it stands in the table.
        color = QtGui.QColor()
        for x in range(width):
            (r,g,b,a) = self.gradient_table.get_pos_rgba_color_lerped(float(x)/(width-1))
            color.setRgb(int(255*r), int(255*g), int(255*b))
            painter.setPen(color)
            brush.setColor(color)
            painter.drawLine(x, start_y, x, end_y)
        if xform:
            # paint the scaled gradient below
            end_y = start_y
            start_y = 0
            for x in range(width):
                f = float(x)/(width-1)
                (r,g,b,a) = self.gradient_table.get_pos_rgba_color_lerped(xform(f))
                color.set(int(255*r), int(255*g), int(255*b))
                brush.setColor(color)
                painter.drawLine(x, start_y, x, end_y)

##########################################################################
# `Channel` class.
##########################################################################
class Channel(ChannelBase):
    def paint(self, painter):
        """Paint current channel into Canvas (a canvas of a function control
        object).

        Contents of the canvas are not deleted prior to painting,
        so more than one channel can be painted into the same canvas.
        """

        table = self.control.table
        # only control points which are active for the current channel
        # are to be painted. filter them out.
        relevant_control_points = [
            x for x in table.control_points if self.name in x.active_channels
        ]
        # lines between control points
        color = QtGui.QColor(*self.rgb_color)
        painter.setPen(color)
        brush = QtGui.QBrush(color)
        painter.setBrush(brush)
        painter.setBackgroundMode(QtCore.Qt.OpaqueMode)
        for k in range( len(relevant_control_points) - 1 ):
            cur_point = relevant_control_points[k]
            next_point = relevant_control_points[1+k]

            painter.drawLine(self.get_pos_index(cur_point.pos),
                             self.get_value_index(cur_point.color),
                             self.get_pos_index(next_point.pos),
                             self.get_value_index(next_point.color))

        # control points themself.
        color = QtCore.Qt.black
        painter.setPen(color)
        for control_point in relevant_control_points:
            x = self.get_pos_index( control_point.pos )
            y = self.get_value_index( control_point.color )
            radius=6
            #print(x,y)
            painter.drawRect(x-(radius/2.0), y-(radius/2.0), radius, radius)
            painter.drawRect(100,80,6,6)

##########################################################################
# `QFunctionControl` class.
##########################################################################
class QFunctionControl(QtGui.QWidget, FunctionControl):
    """Widget which displays a rectangular regions on which hue, sat, val
    or rgb values can be modified. An function control can have one or more
    attached color channels."""

    # Radius around a control point center in which we'd still count a
    # click as "clicked the control point"
    control_pt_click_tolerance = 4

    ChannelFactory = Channel

    def __init__(self, master=None, gradient_table=None, color_space=None,
                 width=100, height=100):
        """Initialize a function control widget on tkframe master.

        Parameters:
        -----------

        master: The master widget.  Note that this widget *must* have
             the methods specified in the `AbstractGradientEditorWidget`
             interface.

        on_table_changed: Callback function taking a bool argument of meaning
            'FinalUpdate'. FinalUpdate is true if a control point is dropped,
            created or removed and false if the update is due to a control point
            currently beeing dragged (but not yet dropped)

        color_space: String which specifies the channels painted on this control.
             May be any combination of h,s,v,r,g,b,a in which each channel
             occurs only once.

        set_status_text: a callback used to set the status text
             when using the editor.
        """
        kw = dict(
            master=master, gradient_table=gradient_table,
            color_space=color_space, width=width,
            height=height
        )
        super().__init__(**kw)
        self.resize(width, height)
        self.setMinimumSize(100, 50)

    ######################################################################
    # Qt event handlers.
    ######################################################################
    def paintEvent(self, event):
        super(QFunctionControl, self).paintEvent(event)

        painter = QtGui.QPainter(self)
        brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
        painter.setBrush(brush)
        width, height = self.size().width(), self.size().height()
        painter.drawRect(0, 0, width, height)
        for channel in self.channels:
            channel.paint(painter)

    def mousePressEvent(self, event):
        if event.button() == QtCore.Qt.LeftButton:
            self.cur_drag = self.find_control_point(event.x(), event.y())
        super(QFunctionControl, self).mousePressEvent(event)

    def mouseReleaseEvent(self, event):
        if event.button() == QtCore.Qt.LeftButton:
            if self.cur_drag:
                self.table_config_changed( final_update = True )
                self.cur_drag = None
        elif event.button() == QtCore.Qt.RightButton:
            # toggle control point. check if there is a control point
            # under the mouse. If yes, delete it, if not, create one
            # at that point.
            cur_control_point = self.find_control_point(event.x(), None)
            if cur_control_point:
                # found a marker at the click position. delete it and return,
                # unless it is a fixed marker (at pos 0 or 1)..
                if ( cur_control_point[1].fixed ):
                    # in this case do nothing. Fixed markers cannot be deleted.
                    return
                self.table.control_points.remove(cur_control_point[1])
                self.table_config_changed(final_update=True)
            else:
                # since there was no marker to remove at the point, we assume
                # that we should place one there
                new_control_point = ColorControlPoint(active_channels=self.active_channels_string)
                new_control_point.set_pos(self.channels[0].get_index_pos(event.x()))

                # set new control point color to the color currently present
                # at its designated position
                new_control_point.color = self.table.get_pos_color(new_control_point.pos)

                self.table.insert_control_point(new_control_point)
                self.table_config_changed(final_update = True)

        if isinstance(event, QtGui.QMouseEvent):
            super(QFunctionControl, self).mouseReleaseEvent(event)

    def leaveEvent(self, event):
        if self.cur_drag:
            self.table_config_changed( final_update = True )
            self.cur_drag = None
        super(QFunctionControl, self).leaveEvent(event)

    def resizeEvent(self, event):
        sz = self.size()
        self.width = sz.width()
        self.height = sz.height()

    def mouseMoveEvent(self, event):
        # currently dragging a control point?
        channel = None
        point = None
        if self.cur_drag:
            channel = self.cur_drag[0]
            point = self.cur_drag[1]
            if ( not point.fixed ):
                point.set_pos( channel.get_index_pos(event.x()) )
                point.activate_channels( self.active_channels_string )
                self.table.sort_control_points()
            channel.set_value_index( point.color, event.y() )
            self.table_config_changed( final_update = False )

        screenX = event.x()
        screenY = event.y()
        width, height = self.size().width(), self.size().height()
        master = self.master
        s1, s2 = master.get_table_range()
        if channel is not None:
            name = self.text_map[channel.name]
            pos = s1 + (s2 - s1)*point.pos
            val = channel.get_value(point.color)
            txt = '%s: (%.3f, %.3f)'%(name, pos, val)
        else:
            x = s1 + (s2 - s1)*float(screenX)/(width-1)
            y = 1.0 - float(screenY)/(height-1)
            txt = "position: (%.3f, %.3f)"%(x, y)

        self.master.set_status_text(txt)

##########################################################################
# `QGradientEditorWidget` class.
##########################################################################
class QGradientEditorWidget(QtGui.QWidget, GradientEditorWidget):
    """A Gradient Editor widget that can be used anywhere.
    """
    def __init__(self, master, vtk_table, on_change_color_table=None,
                 colors=None):
        """

        Parameters:
        -----------

        vtk_table : the `tvtk.LookupTable` or `tvtk.VolumeProperty` object
                    to set.

        on_change_color_table : A callback called when the color table
                                changes.

        colors : list of 'rgb', 'hsv', 'h', 's', 'v', 'a'
                 (Default : ['rgb', 'hsv', 'a'])

                 'rgb' creates one panel to edit Red, Green and Blue
                 colors.

                 'hsv' creates one panel to edit Hue, Saturation and
                 Value.

                 'h', 's', 'v', 'r', 'g', 'b', 'a' separately
                 specified creates different panels for each.
        """
        kw = dict(master=master, vtk_table=vtk_table,
                    on_change_color_table=on_change_color_table,
                    colors=colors)
        super().__init__(**kw)

        gradient_preview_width = self.gradient_preview_width
        gradient_preview_height = self.gradient_preview_height
        channel_function_width = self.channel_function_width
        channel_function_height = self.channel_function_height

        # set up all the panels in a grid
        # 6x2 size:  6 rows, 2 columns...
        grid = QtGui.QGridLayout()
        grid.setColumnStretch(0, 0)
        grid.setColumnStretch(1, 1)

        # "Gradient Viewer" panel, in position (0,1) for sizer
        self.gradient_control = QGradientControl(self,
                                                 self.gradient_table,
                                                 gradient_preview_width,
                                                 gradient_preview_height)
        self.setToolTip('Right click for menu')
        grid.addWidget(QtGui.QLabel("", self), 0, 0)
        grid.addWidget(self.gradient_control, 0, 1)

        # Setup the context menu to fire for the Gradient control alone.
        gc = self.gradient_control
        gc.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
        gc.customContextMenuRequested.connect(self.contextMenuEventOnGradient)

        # Add the function controls:
        function_controls = self.function_controls
        editor_data = self.editor_data

        row = 1
        for color in self.colors:
            data = editor_data[color]
            control = QFunctionControl(self, self.gradient_table, color,
                                       channel_function_width,
                                       channel_function_height)
            txt = data[0] + self.tooltip_text
            control.setToolTip(txt)
            # Add name of editor (to left side of editor)
            grid.addWidget(QtGui.QLabel(data[1], self), row, 0)
            # Add the "RGB" control point editor
            grid.addWidget(control, row, 1)
            function_controls.append(control)
            row += 1

        # The status text.
        self.text = QtGui.QLabel('status', self)
        grid.addWidget(self.text, row, 0, 1, 2)

        self.setLayout(grid)
        self.show()

    ######################################################################
    # `GradientEditorWidget` interface.
    ######################################################################
    def set_status_text(self, msg):
        self.text.setText(msg)

    ######################################################################
    # Qt event methods.
    ######################################################################
    def contextMenuEventOnGradient(self, pos):
        menu = QtGui.QMenu(self)
        saveAction = menu.addAction("Save as")
        loadAction = menu.addAction("Load")
        action = menu.exec_(self.mapToGlobal(pos))
        if action == saveAction:
            self.on_save()
        elif action == loadAction:
            self.on_load()

    def on_save(self, event=None):
        """
        Open "Save" dialog, write lookuptable to 3 files: ``*.lut``
        (lookuptable) ``*.grad`` (gradient table for use with this program),
        and ``*.jpg`` (image of the gradient)
        """
        wildcard = "Gradient Files (*.grad);;All Files (*.*)"
        filename, filter = QtGui.QFileDialog.getSaveFileName(self,
                                                "Save LUT to...",
                                                '',
                                                wildcard)
        if filename:
            self.save(filename)

    def on_load(self, event=None):
        """
        Load a ``*.grad`` lookuptable file.
        """
        wildcard = "Gradient Files (*.grad);;All Files (*.*)"
        filename, filter = QtGui.QFileDialog.getOpenFileName(self,
                                                "Open gradient file...",
                                                '',
                                                wildcard)
        if filename:
            self.load(filename)

##########################################################################
# `QGradientEditor` class.
##########################################################################
class QGradientEditor(QtGui.QMainWindow):
    """ QMainWindow that displays the gradient editor window,
    i.e. the thing that contains the gradient display, the function
    controls and the buttons.
    """
    def __init__(self, vtk_table, on_change_color_table=None, colors=None):
        """Initialize the gradient editor window.

        Parameters
        ----------
        vtk_table: Instance of vtkLookupTable, designating the table which is
            to be edited.
        on_change_color_table: Callback function taking no arguments. Called
            when the color table was changed and rendering is
            requested.
        """
        super(QGradientEditor, self).__init__()
        self.setWindowTitle("Color Gradient Editor")
        self.widget = QGradientEditorWidget(
            master=self, vtk_table=vtk_table,
            on_change_color_table=on_change_color_table,
            colors=colors
        )

        self.setCentralWidget(self.widget)
        self.resize(300, 500)

        self.statusBar()
        ## Set up the MenuBar
        menu = self.menuBar()
        file_menu = menu.addMenu("&File")

        file_action = QtGui.QAction("&Save", self)
        file_action.setStatusTip("Save CTF")
        file_action.triggered.connect(self.widget.on_save)
        file_menu.addAction(file_action)

        load_action = QtGui.QAction("&Load", self)
        load_action.setStatusTip("Load CTF")
        load_action.triggered.connect(self.widget.on_load)
        file_menu.addAction(load_action)

        quit_action = QtGui.QAction("&Quit", self)
        quit_action.setStatusTip("Quit application")
        quit_action.triggered.connect(QtGui.QApplication.instance().quit)
        file_menu.addAction(quit_action)

        help_menu = menu.addMenu("&Help")
        action = QtGui.QAction("&Help", self)
        action.setStatusTip("Help")
        action.triggered.connect(self.on_help)
        help_menu.addAction(action)
        action = QtGui.QAction("&About", self)
        action.setStatusTip("About application")
        action.triggered.connect(self.on_about)
        help_menu.addAction(action)

    def on_help(self, event=None):
        """ Help defining the mouse interactions """
        message = "Right click to add control points.  Left click to move control points"
        QtGui.QMessageBox.information(self, 'Help', message)

    def on_about(self, event=None):
        """ Who wrote the program?"""
        message = 'tk Gradient Editor for MayaVi1: Gerald Knizia (cgk.d@gmx.net)\n'\
                  'wxPython port: Pete Schmitt (schmitt@colorado.edu)\n'\
                  'Qt port: Prabhu Ramachandran\n'\
                  'Enhanced for Mayavi2: Prabhu Ramachandran'
        QtGui.QMessageBox.information(self, 'About gradient editor', message)

def main():
    from tvtk.util.traitsui_gradient_editor import make_test_table
    import sys
    table, ctf, otf = make_test_table(lut=False)
    # the actual gradient editor code.
    def on_color_table_changed():
        """If we had a vtk window running, update it here"""
        # print("Update Render Window")
        pass

    app = QtGui.QApplication.instance()
    editor = QGradientEditor(table,
                             on_color_table_changed,
                             colors=['rgb', 'a', 'h', 's', 'v'],
    )
    editor.setWindowTitle("Gradient editor")
    editor.show()
    sys.exit(app.exec_())

##########################################################################
# Test application.
##########################################################################
if __name__ == "__main__":
    main()


import logging
from functools import partial

from datasets import load_dataset
from transformers import (
    Seq2SeqTrainer,
    Seq2SeqTrainingArguments,
    WhisperForConditionalGeneration,
    WhisperProcessor,
)

from src.callbacks import ShuffleCallback
from src.config import Config, TrainingArgumentsConfig
from src.data_collator import DataCollatorSpeechSeq2SeqWithPadding
from src.metrics import compute_metrics
from src.prepare_dataset import prepare_dataset

logging.basicConfig(level=logging.INFO)

def train():
    config = Config()
    training_args_config = TrainingArgumentsConfig()
    training_args = Seq2SeqTrainingArguments(**training_args_config.dict())

    if config.prepare_dataset:
        dataset, _ = prepare_dataset(config)
    else:
        dataset = load_dataset(config.dataset_name, config.dataset_lang)
    logging.info("Training model...")

    model = WhisperForConditionalGeneration.from_pretrained(config.model_name)
    processor = WhisperProcessor.from_pretrained(
        config.model_name, task=config.task, language=config.model_lang
    )
    compute_metrics_fn = partial(compute_metrics, processor=processor)
    trainer = Seq2SeqTrainer(
        args=training_args,
        model=model,
        train_dataset=dataset["train"],
        eval_dataset=dataset["validation"],
        data_collator=DataCollatorSpeechSeq2SeqWithPadding(processor=processor),
        compute_metrics=compute_metrics_fn,
        tokenizer=processor,
        callbacks=[ShuffleCallback()],
    )

    trainer.train()
    trainer.push_to_hub()

if __name__ == "__main__":
    train()


from booleano.exc import InvalidOperationError
from booleano.operations.operands import Operand

__all__ = ["String", "Number", "Arithmetic", "Set"]

class Constant(Operand):
	"""
	Base class for constant operands.

	The only operation that is common to all the constants is equality (see
	:meth:`equals`).

	Constants don't rely on the context -- they are constant!

	.. warning::
		This class is available as the base for the built-in :class:`String`,
		:class:`Number` and :class:`Set` classes. User-defined constants aren't
		supported, but you can assign a name to a constant (see
		:term:`binding`).

	"""

	operations = {'equality'}

	def __init__(self, constant_value):
		"""

		:param constant_value: The Python value represented by the Booleano
			constant.
		:type constant_value: :class:`object`

		"""
		self.constant_value = constant_value

	def to_python(self, context):
		"""
		Return the value represented by this constant.

		"""
		return self.constant_value

	def equals(self, value, context):
		"""
		Check if this constant equals ``value``.

		"""
		return self.constant_value == value

	def check_equivalence(self, node):
		"""
		Make sure constant ``node`` and this constant are equivalent.

		:param node: The other constant which may be equivalent to this one.
		:type node: Constant
		:raises AssertionError: If the constants are of different types or
			represent different values.

		"""
		super(Constant, self).check_equivalence(node)
		assert node.constant_value == self.constant_value, \
			u'Constants %s and %s represent different values' % (self,
			                                                     node)

class String(Constant):
	u"""
	Constant string.

	These constants only support equality operations.

	.. note:: **Membership operations aren't supported**

		Although both sets and strings are item collections, the former is
		unordered and the later is ordered. If they were supported, there would
		some ambiguities to sort out, because users would expect the following
		operation results:

		- ``"ao" ⊂ "hola"`` is false: If strings were also sets, then the
		  resulting operation would be ``{"a", "o"} ⊂ {"h", "o", "l", "a"}``,
		  which is true.
		- ``"la" ∈ "hola"`` is true: If strings were also sets, then the
		  resulting operation would be ``{"l", "a"} ∈ {"h", "o", "l", "a"}``,
		  which would be an *invalid operation* because the first operand must
		  be an item, not a set. But if we make an exception and take the first
		  operand as an item, the resulting operation would be
		  ``"la" ∈ {"h", "o", "l", "a"}``, which is not true.

		The solution to the problems above would involve some magic which
		contradicts the definition of a set: Take the second operand as an
		*ordered collection*. But it'd just cause more trouble, because both
		operations would be equivalent!

		Also, there would be other issues to take into account (or not), like
		case-sensitivity.

		Therefore, if this functionality is needed, developers should create
		functions to handle it.

	"""

	def __init__(self, string):
		"""

		:param string: The Python string to be represented by this Booleano
			string.
		:type string: :class:`basestring`

		``string`` will be converted to :class:`unicode`, so it doesn't
		have to be a :class:`basestring` initially.

		"""
		import sys
		if sys.version_info >= (3, 0):
			string = str(string)
		else:
			string = unicode(string)
		super(String, self).__init__(string)

	def equals(self, value, context):
		"""Turn ``value`` into a string if it isn't a string yet"""
		value = str(value)
		return super(String, self).equals(value, context)

	def __unicode__(self):
		"""Return the Unicode representation of this constant string."""
		return u'"%s"' % self.constant_value

	def __hash__(self):
		return id(self)

	def __repr__(self):
		"""Return the representation for this constant string."""
		return '<String "%s">' % self.constant_value.encode("utf-8")

class ArithmeticVariable(object):
	def __init__(self, number, namespace, namespace_separator=":"):
		self.namespace_separator = namespace_separator
		self.parsed_results = number
		self._namespace = namespace
		self.variables = {}
		self.__define_variables()
		number = self.flatten(self.parsed_results)
		self.__full_expression = "".join(number)

	def __str__(self):
		number = self.flatten(self.parsed_results)
		return "".join(number)

	def __define_variables(self):
		number = self.parsed_results
		temp = []
		for n in number:
			t = self.__get_variable_names(n)
			if isinstance(t, list):
				temp.extend(t)
			else:
				temp.append(t)
		self.required_variables = temp
		temp = {}
		for v in self.required_variables:
			for k, val in v.items():
				temp[k] = val
		self.required_variables = temp

	def __get_variable_names(self, number):
		from pyparsing import ParseResults
		import re
		temp = []
		if isinstance(number, ParseResults):
			for n in number:
				t = self.__get_variable_names(n)
				if isinstance(t, list):
					temp.extend(t)
				else:
					temp.append(t)
			return temp
		elif len(re.findall("[a-zA-Z" + self.namespace_separator + "]+", number)) > 0:
			var = str(number).split(self.namespace_separator)
			variable_namespaces = var[0:-1]
			variable_name = var[-1]
			return {str(number): self._namespace.get_object(variable_name, variable_namespaces)}
		return temp

	@classmethod
	def flatten(cls, s):
		from pyparsing import ParseResults
		if s == []:
			return s
		if isinstance(s[0], ParseResults):
			return cls.flatten(s[0]) + cls.flatten(s[1:])
		return s[:1] + cls.flatten(s[1:])

	def replace(self, num, context, namespace=True):
		for k, v in self.required_variables.items():
			if namespace and self.namespace_separator not in k:
				continue
			num = num.replace(k, str(v.to_python(context)))
		return num

	def evaluate(self, context):
		number = self.__full_expression
		# Replace all variables with numbers
		# First replace variables with namespaces defined to avoid clobbering
		number = self.replace(number, context)
		# Then replace variables with no namespace
		number = self.replace(number, context, False)
		number = number.replace("^", "**")
		from booleano import SafeEval
		answer = SafeEval.eval_expr(number)
		return answer

class Arithmetic(Constant):
	"""
	Numeric constant.

	These constants support inequality operations; see :meth:`greater_than`
	and :meth:`less_than`.

	"""

	operations = Constant.operations | {'inequality'}

	def __init__(self, number, namespace, namespace_separator=":"):
		"""

		:param number: The number to be represented, as a Python object.
		:type number: :class:`object`

		``number`` is converted into a :class:`float` internally, so it can
		be an :class:`string <basestring>` initially.

		"""
		self.namespace_sparator = namespace_separator
		super(Arithmetic, self).__init__(ArithmeticVariable(number, namespace, namespace_separator))

	def equals(self, value, context):
		"""
		Check if this numeric constant equals ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		print("Constant equals")
		return super(Arithmetic, self).equals(self._to_number(value), context)

	def greater_than(self, value, context):
		"""
		Check if this numeric constant is greater than ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		print("Constant gt")
		return self.constant_value > self._to_number(value)

	def less_than(self, value, context):
		"""
		Check if this numeric constant is less than ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		print("Constant lt")
		return self.constant_value < self._to_number(value)

	def to_python(self, context):
		return self.constant_value.evaluate(context)

	def _to_number(self, value):
		"""
		Convert ``value`` to a Python float and return the new value.

		:param value: The value to be converted into float.
		:return: The value as a float.
		:rtype: float
		:raises InvalidOperationError: If ``value`` can't be converted.

		"""
		print("Constant to_num")
		try:
			return float(value)
		except ValueError:
			raise InvalidOperationError('"%s" is not a number' % value)

	def __unicode__(self):
		"""Return the Unicode representation of this constant number."""
		print("constant unicode")
		return str(self.constant_value)

	def __repr__(self):
		"""Return the representation for this constant number."""
		return '<Arithmetic %s>' % self.constant_value

class Number(Constant):
	"""
	Numeric constant.

	These constants support inequality operations; see :meth:`greater_than`
	and :meth:`less_than`.

	"""

	operations = Constant.operations | {'inequality'}

	def __init__(self, number):
		"""

		:param number: The number to be represented, as a Python object.
		:type number: :class:`object`

		``number`` is converted into a :class:`float` internally, so it can
		be an :class:`string <basestring>` initially.

		"""
		number = float(number)
		super(Number, self).__init__(number)

	def equals(self, value, context):
		"""
		Check if this numeric constant equals ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		return super(Number, self).equals(self._to_number(value), context)

	def greater_than(self, value, context):
		"""
		Check if this numeric constant is greater than ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		return self.constant_value > self._to_number(value)

	def less_than(self, value, context):
		"""
		Check if this numeric constant is less than ``value``.

		:raises InvalidOperationError: If ``value`` can't be turned into a
			float.

		``value`` will be turned into a float prior to the comparison, to
		support strings.

		"""
		return self.constant_value < self._to_number(value)

	def _to_number(self, value):
		"""
		Convert ``value`` to a Python float and return the new value.

		:param value: The value to be converted into float.
		:return: The value as a float.
		:rtype: float
		:raises InvalidOperationError: If ``value`` can't be converted.

		"""
		try:
			return float(value)
		except ValueError:
			raise InvalidOperationError('"%s" is not a number' % value)

	def __unicode__(self):
		"""Return the Unicode representation of this constant number."""
		return str(self.constant_value)

	def __repr__(self):
		"""Return the representation for this constant number."""
		return '<Number %s>' % self.constant_value

class Set(Constant):
	"""
	Constant sets.

	These constants support membership operations; see :meth:`contains` and
	:meth:`is_subset`.

	"""

	operations = Constant.operations | {"inequality", "membership"}

	def __init__(self, *items):
		"""

		:raises booleano.exc.InvalidOperationError: If at least one of the
			``items`` is not an operand.

		"""
		for item in items:
			if not isinstance(item, Operand):
				raise InvalidOperationError('Item "%s" is not an operand, so '
				                            'it cannot be a member of a set' %
				                            item)
		super(Set, self).__init__(set(items))

	def to_python(self, context):
		"""
		Return a set made up of the Python representation of the operands
		contained in this set.

		"""
		items = set(item.to_python(context) for item in self.constant_value)
		return items

	def equals(self, value, context):
		"""Check if all the items in ``value`` are the same of this set."""
		value = set(value)
		return value == self.to_python(context)

	def less_than(self, value, context):
		"""
		Check if this set has less items than the number represented in
		``value``.

		:raises InvalidOperationError: If ``value`` is not an integer.

		"""
		value = self._to_int(value)
		return len(self.constant_value) < value

	def greater_than(self, value, context):
		"""
		Check if this set has more items than the number represented in
		``value``.

		:raises InvalidOperationError: If ``value`` is not an integer.

		"""
		value = self._to_int(value)
		return len(self.constant_value) > value

	def belongs_to(self, value, context):
		"""
		Check that this constant set contains the ``value`` item.

		"""
		for item in self.constant_value:
			try:
				if item.equals(value, context):
					return True
			except InvalidOperationError:
				continue
		return False

	def is_subset(self, value, context):
		"""
		Check that the ``value`` set is a subset of this constant set.

		"""
		for item in value:
			if not self.belongs_to(item, context):
				return False
		return True

	def check_equivalence(self, node):
		"""
		Make sure set ``node`` and this set are equivalent.

		:param node: The other set which may be equivalent to this one.
		:type node: Set
		:raises AssertionError: If ``node`` is not a set or it's a set
			with different elements.

		"""
		Operand.check_equivalence(self, node)

		unmatched_elements = list(self.constant_value)
		assert len(unmatched_elements) == len(node.constant_value), \
			u'Sets %s and %s do not have the same cardinality' % \
			(unmatched_elements, node)

		# Checking that each element is represented by a mock operand:
		for element in node.constant_value:
			for key in range(len(unmatched_elements)):
				if unmatched_elements[key] == element:
					del unmatched_elements[key]
					break

		assert 0 == len(unmatched_elements), \
			u'No match for the following elements: %s' % unmatched_elements

	def __unicode__(self):
		"""Return the Unicode representation of this constant set."""
		elements = [str(element) for element in self.constant_value]
		elements = u", ".join(elements)
		return "{%s}" % elements

	def __repr__(self):
		"""Return the representation for this constant set."""
		elements = [repr(element) for element in self.constant_value]
		elements = ", ".join(elements)
		if elements:
			elements = " " + elements
		return '<Set%s>' % elements

	@classmethod
	def _to_int(cls, value):
		"""
		Convert ``value`` is to integer if possible.

		:param value: The value to be verified.
		:return: ``value`` as integer.
		:rtype: int
		:raises InvalidOperationError: If ``value`` is not an integer.

		This is a workaround for Python < 2.6, where floats didn't have the
		``.is_integer()`` method.

		"""
		try:
			value_as_int = int(value)
			is_int = value_as_int == float(value)
		except (ValueError, TypeError):
			is_int = False
		if not is_int:
			raise InvalidOperationError("To compare the amount of items in a "
			                            "set, the operand %s has to be an "
			                            "integer" % repr(value))
		return value_as_int

{'ParseResults': 'pyparsing.ParseResults', 're': 're', 'SafeEval': 'booleano.SafeEval'}


import numpy as np
import matplotlib.pyplot as plt

def radial_kernel(x0, X, tau):
    return np.exp(np.sum((X - x0) ** 2, axis=1) / (-2 * tau * tau))

def local_regression(x0, X, Y, tau):
    # add bias term
    x0 = np.r_[1, x0]
    X = np.c_[np.ones(len(X)), X]
    # fit model: normal equations with kernel
    xw = X.T * radial_kernel(x0, X, tau)
    beta = np.linalg.pinv(xw @ X) @ xw @ Y
    # predict value
    return x0 @ beta

def generate_data():
    n = 1000
    X = np.linspace(-3, 3, num=n)
    Y = np.log(np.abs(X ** 2 - 1) + .5)
    # Y = np.sin(X) + 0.3 * np.random.randn(n)  #
    plt.scatter(X, Y, s=5, color="green")
    plt.savefig("LocalWeightedLinearRegression2-DataInitial.png")
    plt.cla()  # Clear axis
    plt.clf()  # Clear figure
    plt.close()  # Close a figure window
    # jitter X
    X += np.random.normal(scale=.1, size=n)
    plt.scatter(X, Y, s=5, color="green")
    plt.savefig("LocalWeightedLinearRegression2-DatawithGitter.png")
    plt.cla()  # Clear axis
    plt.clf()  # Clear figure
    plt.close()  # Close a figure window
    return X, Y

def create_plot(X, Y, tau):
    fig, axes = plt.subplots(3, 2, figsize=(16, 8), sharex=False, sharey=False, dpi=120)
    # plt.subplots(3, 2 ) means display data in 3 rows and 2 columns
    # Plot each axes
    for i, ax in enumerate(axes.ravel()):
        domain = np.linspace(-3, 3, num=40)
        prediction = [local_regression(x0, X, Y, tau[i]) for x0 in domain]
        ax.scatter(X, Y, s=5, color="green", label="actual")
        ax.scatter(domain, prediction, s=5, color='red', label="prediction")
        ax.set(
            title="tau=" + str(tau[i]),
            xlabel='X',
            ylabel='Y',
         )
        ax.legend(loc='best')
    plt.suptitle('Local Weight Linear regression', size=10, color='blue')
    plt.savefig("LocalWeightedLinearRegression2-DataAndPrediction.png")
    return plt

if __name__ == "__main__":
    X, Y = generate_data()
    tau = [800, 10, .1, .01, .08, .9]
    myplot = create_plot(X, Y, tau)
    myplot.show()


########################################################################################
# Run examples from our paper in RBEF
########################################################################################
import sys
import numpy             as     np
import matplotlib.pyplot as     plt
import multiprocessing
import scipy
import scipy.signal
from   scipy.integrate   import simps
from   joblib            import Parallel, delayed
from   ar_model          import *
import pygc.pySpec
import pygc.parametric
import pygc.non_parametric
import pygc.granger
import plot_results

p = int(sys.argv[-1])

if p == 0:
	# Generates figure 3 from the paper

	print('Generating Figure 3 from RBEF paper...')

	N  = 5000      # Number of observations
	Fs = 200       # Sampling frequency
	dt = 1.0 / Fs  # Time resolution
	C  = 0.25      # Coupling parameter
	Trials = 5000  # Number of trials
	# Covariance matrix
	cov = np.array([ [1.00, 0.00],
					 [0.00, 1.00] ])

	f = pygc.pySpec.compute_freq(N, Fs)

	S = np.zeros([2,2,N//2+1]) + 1j*np.zeros([2,2,N//2+1])

	print('Generating AR model time series...')
	Z = ar_model_dhamala(N=N, Trials = Trials, C=C, Fs=Fs, t_start=0, t_stop=None, cov=cov)

	print('Estimating spectral matrix from ' + str(Trials) + ' trials...')
	for i in range(Trials):
		if i%500 == 0:
			print('Trial = ' + str(i))

		S[0,0] += pygc.pySpec.cxy(X=Z[0,i,:], Y=[], f=f, Fs=Fs) / Trials
		S[0,1] += pygc.pySpec.cxy(X=Z[0,i,:], Y=Z[1,i,:], f=f, Fs=Fs) / Trials
		S[1,0] += pygc.pySpec.cxy(X=Z[1,i,:], Y=Z[0,i,:], f=f, Fs=Fs) / Trials
		S[1,1] += pygc.pySpec.cxy(X=Z[1,i,:], Y=[], f=f, Fs=Fs) / Trials

	print('Computing Granger Causality...')
	Snew, Hnew, Znew = pygc.non_parametric.wilson_factorization(S, f, Fs, Niterations=30)
	Ix2y, Iy2x, Ixy  = pygc.granger.granger_causality(S, Hnew, Znew)

	print('Saving data...')
	np.save('data/fig3.npy', {'f': f, 'S': S, 'H': Hnew, 'Z': Znew, 'Ix2y': Ix2y, 'Iy2x': Iy2x, 'Ixy': Ixy})

	print('Plotting results...')
	plot_results.fig3()

if p == 1:
	# Generates figure 4 from the paper

	N  = 900      # Number of observations
	Fs = 200      # Sampling frequency
	dt = 1.0 / Fs # Time resolution
	C  = 0.25     # Coupling parameter
	Trials = 5000 # Number of trials

	cov = np.array([ [1.00, 0.00],
					 [0.00, 1.00] ])

	f = pygc.pySpec.compute_freq(N, Fs)

	S = np.zeros([2,2,N,N//2+1]) + 1j*np.zeros([2,2,N,N//2+1])

	print('Generating AR model time series...')
	Z = ar_model_dhamala(N=N, Trials = Trials, C=C, Fs=Fs, t_start=0, t_stop=2.25, cov=cov)

	print('Estimating wavelet matrix from ' + str(Trials) + ' trials...')
	for i in range(Trials):
		if i%500 == 0:
			print('Trial = ' + str(i))

		Wx = pygc.pySpec.morlet(Z[0,i,:], f, Fs)
		Wy = pygc.pySpec.morlet(Z[1,i,:], f, Fs)

		S[0,0] += Wx*np.conj(Wx) / Trials
		S[0,1] += Wx*np.conj(Wy) / Trials
		S[1,0] += Wy*np.conj(Wx) / Trials
		S[1,1] += Wy*np.conj(Wy) / Trials

#	S   = S[:,:,idx,:]
	print('Computing Granger Causality...')
	def save_granger(S, idx):
		Snew, Hnew, Znew = pygc.non_parametric.wilson_factorization(S[:,:,idx,:], f, Fs, Niterations=30, verbose=False)
		Ix2y, Iy2x, Ixy  = pygc.granger.granger_causality(S[:,:,idx,:], Hnew, Znew)
		np.save('data/fig4_'+str(idx)+'.npy', {'f': f, 'Ix2y': Ix2y, 'Iy2x': Iy2x, 'Ixy': Ixy})

	Parallel(n_jobs=40,	 backend='loky', max_nbytes=1e6)(delayed(save_granger)(S, idx) for idx in range(N))
	print('Plotting results...')
	plot_results.fig4()

if p == 2:
	# Generates figure 7 and 8 from the paper
	N       = 5000  # Number of observations
	Trials  = 1000  # Number of trials
	nvars   = 5     # Number of variables

	Fs = 2*np.pi
	dt = 1.0 / Fs
	f  = pygc.pySpec.compute_freq(N, Fs)

	print('Generating AR model time series...')
	Y = ar_model_baccala(nvars, N, Trials)

	print('Estimating spectral matrix from ' + str(Trials) + ' trials...')
	S = np.zeros([nvars, nvars, N//2 + 1]) * (1 + 1j)

	for trial in range(Trials):
		if (trial % 100 == 0):
			print('Trial = ' + str(trial))
		for i in range(nvars):
			for j in range(nvars):
				S[i,j] += pygc.pySpec.cxy(X=Y[i,:,trial], Y=Y[j,:,trial], f=f, Fs=Fs) / Trials

	print('Estimating pairwise Granger casalities')
	GC = np.zeros([nvars, nvars])
	for i in range(nvars):
		for j in range(nvars):
			if i == j:
				continue
			else:
				S_aux = np.array([[S[i,i], S[i,j]],[S[j,i], S[j,j]]])
				_, H, Z = pygc.non_parametric.wilson_factorization(S_aux, f, Fs, Niterations=10, tol=1e-12, verbose=False)
				Ix2y, Iy2x, _ = pygc.granger.granger_causality(S_aux, H, Z)
				GC[i,j] = simps(Ix2y, f) / 2*np.pi
				GC[j,i] = simps(Iy2x, f) / 2*np.pi

	print('Estimating conditional Granger casalities')
	F   = pygc.granger.conditional_granger_causality(S, f, Fs, Niterations = 10, verbose=False)
	cGC = pygc.granger.conditional_spec_granger_causality(S, f, Fs, Niterations=100, tol=1e-12, verbose=False)

	print('Saving data...')
	np.save('data/fig_7_8.npy', {'f':f,'GC': GC, 'F': F, 'cGC': cGC})

	print('Plotting results...')
	plot_results.fig7_8()

if p == 3:
	# Fits an AR model by solving YW equations as in appendix A of the paper.

	Trials = 1000  # Number of trials
	Fs     = 200   # Sampling frequency
	N      = 1000  # Number of data points
	X      = np.zeros([1,N, Trials]) # Data matrix
	tsim   = N/Fs # Simulation time

	# Coefficients of the ar model
	c = [0.7, 0.2, -0.1, -0.3]

	print('Generating AR model time series...')
	for T in range(Trials):
		X[0,:,T] = scipy.signal.lfilter([1], -np.array([-1]+c), np.random.randn(N))

	print('Estimating AR model coefficients for ' + str(Trials) + ' trials')

	for m in [2, 3, 4, 5, 6]:

		print()

		AR  = np.zeros([1,1,m])
		SIG = np.zeros([1,1])
		for T in range(Trials):
			aux1, aux2 = pygc.parametric.YuleWalker(X[:,:,T], m, maxlags=100)
			AR  += aux1.T/Trials
			SIG += aux2.T/Trials

		AR = np.round(AR, 2)
		SIG = np.round(SIG, 2)
		print('Using order = ' + str(m)+ '. Original coefficients: ' + str(c) + '. Estimated coefficients ' + str(AR[0][0]) + '. Noise variace: ' + str(SIG[0][0]))

if p == 4:
	# Generates figure 3C from the paper, but using a paramtreic method
	# Generates figure 3 from the paper

	print('Generating Figure 3 from RBEF paper...')

	N  = 5000      # Number of observations
	Fs = 200       # Sampling frequency
	dt = 1.0 / Fs  # Time resolution
	C  = 0.25      # Coupling parameter
	Trials = 5000  # Number of trials
	# Covariance matrix
	cov = np.array([ [1.00, 0.00],
					 [0.00, 1.00] ])

	print('Generating AR model time series...')
	X = ar_model_dhamala(N=N, Trials = Trials, C=C, Fs=Fs, t_start=0, t_stop=None, cov=cov)

	print('Estimating VAR coefficients using oreder m=2...')
	m   = 2
	AR  = np.zeros([m, 2,2])
	SIG = np.zeros([2,2])
	for T in range(Trials):
		aux1, aux2 = pygc.parametric.YuleWalker(X[:,T,:], m, maxlags=100)
		AR  += aux1/Trials
		SIG += aux2/Trials

	print('Computing Granger Causality...')
	f    = pygc.pySpec.compute_freq(N, Fs)
	H, S = pygc.parametric.compute_transfer_function(AR, SIG, f, Fs)
	Ix2y, Iy2x, _  = pygc.granger.granger_causality(S, H, SIG)

	plt.figure(figsize=(6,2))
	plt.plot(f, Ix2y)
	plt.plot(f, Iy2x)
	plt.xlim([0, 100])
	plt.ylim([-0.01, 1.2])
	plt.ylabel('GC')
	plt.xlabel('Frequency [Hz]')
	plt.legend([r'$X_{1}\rightarrow X_{2}$', r'$X_{2}\rightarrow X_{1}$'])
	plt.savefig('figures/fig9.pdf', dpi = 600)
	plt.close()


import torch
from torch import nn
import torchvision.transforms as T

#######################################################################################
######################################## DRML  ########################################
#######################################################################################

class RegionLayer(nn.Module):
    def __init__(self, in_channels, grid=(8, 8)):
        super(RegionLayer, self).__init__()

        self.in_channels = in_channels
        self.grid = grid

        self.region_layers = dict()

        for i in range(self.grid[0]):
            for j in range(self.grid[1]):
                module_name = 'region_conv_%d_%d' % (i, j)
                self.region_layers[module_name] = nn.Sequential(
                    nn.BatchNorm2d(self.in_channels),
                    nn.ReLU(),
                    nn.Conv2d(in_channels=self.in_channels, out_channels=self.in_channels,
                              kernel_size=3, stride=1, padding=1)
                )
                self.add_module(name=module_name, module=self.region_layers[module_name])

    def forward(self, x):
        """
        :param x:   (b, c, h, w)
        :return:    (b, c, h, w)
        """

        batch_size, _, height, width = x.size()

        input_row_list = torch.split(x, split_size_or_sections=height//(self.grid[0]-1), dim=2)
        output_row_list = []

        for i, row in enumerate(input_row_list):
            input_grid_list_of_a_row = torch.split(row, split_size_or_sections=width//(self.grid[1]-1), dim=3)
            output_grid_list_of_a_row = []

            for j, grid in enumerate(input_grid_list_of_a_row):
                module_name = 'region_conv_%d_%d' % (i, j)
                # print(module_name)
                # print(i,j)
                grid = self.region_layers[module_name](grid.contiguous()) + grid
                output_grid_list_of_a_row.append(grid)

            output_row = torch.cat(output_grid_list_of_a_row, dim=3)
            output_row_list.append(output_row)

        output = torch.cat(output_row_list, dim=2)
        return output

class DRML(nn.Module):
    def __init__(self, class_number=12):
        super(DRML, self).__init__()

        print('Init DRML... pls god work...')

        self.class_number = class_number

        self.extractor = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=11, stride=1),
            RegionLayer(in_channels=32, grid=(8, 8)),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.BatchNorm2d(num_features=32),

            nn.Conv2d(in_channels=32, out_channels=16, kernel_size=8, stride=1),
            nn.ReLU(),

            nn.Conv2d(in_channels=16, out_channels=16, kernel_size=8,),
            nn.ReLU(),

            nn.Conv2d(in_channels=16, out_channels=16, kernel_size=6, stride=2),
            nn.ReLU(),

            nn.Conv2d(in_channels=16, out_channels=16, kernel_size=5, stride=1),
            nn.ReLU(),
        )

        self.classifier = nn.Sequential(
            nn.Linear(in_features=6400, out_features=4096),
            nn.ReLU(),
            nn.Dropout(0.5),

            nn.Linear(in_features=4096, out_features=2048),
            nn.ReLU(),
            nn.Dropout(0.5),

            nn.Linear(in_features=2048, out_features=class_number)
        )

    def forward(self, data):
        """
        :param x:   (b, c, h, w)
        :return:    (b, class_number)
        """

        x = data[0]
        batch_size = x.size(0)
        output = self.extractor(x)
        output = output.view(batch_size, -1)
        output = self.classifier(output)
        return output

#######################################################################################
####################################### AlexNet  ######################################
#######################################################################################
class AlexNet(nn.Module):
    def __init__(self, num_classes = 12, dropout = 0.5): #0.5
        super().__init__()

        print('INIT AU AlexNet')

        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=False),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=False),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Dropout(p=dropout),
            nn.Linear(2304, 4096), # assuming 144x144 input
            nn.ReLU(inplace=False),
            nn.Dropout(p=dropout),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=False),
            nn.Linear(4096, num_classes),
        )

    def forward(self, data):

        x = data[0]
        x = self.features(x)
        x = torch.flatten(x, 1)
        out = self.classifier(x)
        return out

#######################################################################################
####################################### MTTSCAN  ######################################
#######################################################################################

class Attention_mask(nn.Module):
    def __init__(self):
        super(Attention_mask, self).__init__()

    def forward(self, x):
        xsum = torch.sum(x, dim=2, keepdim=True)
        xsum = torch.sum(xsum, dim=3, keepdim=True)
        xshape = tuple(x.size())
        return x / xsum * xshape[2] * xshape[3] * 0.5

    def get_config(self):
        """May be generated manually. """
        config = super(Attention_mask, self).get_config()
        return config

class TSM(nn.Module):
    def __init__(self, n_segment=10, fold_div=3):
        super(TSM, self).__init__()
        self.n_segment = n_segment
        self.fold_div = fold_div

    def forward(self, x):
        nt, c, h, w = x.size()
        n_batch = nt // self.n_segment
        x = x.view(n_batch, self.n_segment, c, h, w)
        fold = c // self.fold_div
        out = torch.zeros_like(x)
        out[:, :-1, :fold] = x[:, 1:, :fold]  # shift left
        out[:, 1:, fold: 2 * fold] = x[:, :-1, fold: 2 * fold]  # shift right
        out[:, :, 2 * fold:] = x[:, :, 2 * fold:]  # not shift
        return out.view(nt, c, h, w)

class MTTS_CAN_SMALL(nn.Module):
    """MTTS_CAN is the multi-task (respiration) version of TS-CAN"""

    def __init__(self, in_channels=3, nb_filters1=32, nb_filters2=64, kernel_size=3, dropout_rate1=0.25,
                 dropout_rate2=0.5, pool_size=(2, 2), nb_dense=128, frame_depth=20):
        super(MTTS_CAN_SMALL, self).__init__()

        print('init MTTS_CAN_SMALL')

        self.in_channels = in_channels
        self.kernel_size = kernel_size
        self.dropout_rate1 = dropout_rate1
        self.dropout_rate2 = dropout_rate2
        self.pool_size = pool_size
        self.nb_filters1 = nb_filters1
        self.nb_filters2 = nb_filters2
        self.nb_dense = nb_dense
        # TSM layers
        self.TSM_1 = TSM(n_segment=frame_depth)
        self.TSM_2 = TSM(n_segment=frame_depth)
        self.TSM_3 = TSM(n_segment=frame_depth)
        self.TSM_4 = TSM(n_segment=frame_depth)
        # Motion branch convs
        self.motion_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        # Apperance branch convs
        self.apperance_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        # Attention layers
        self.apperance_att_conv1 = nn.Conv2d(self.nb_filters1, 1, kernel_size=1, padding=(0, 0),bias=True)
        self.attn_mask_1 = Attention_mask()
        self.apperance_att_conv2 = nn.Conv2d(self.nb_filters2, 1, kernel_size=1, padding=(0, 0),bias=True)
        self.attn_mask_2 = Attention_mask()

        # Dropout layers
        self.dropout_4_y = nn.Dropout(self.dropout_rate2)
        self.dropout_4_r = nn.Dropout(self.dropout_rate2)

        # Dense layers
        self.final_dense_1_y = nn.Linear(5184, self.nb_dense, bias=True)
        self.final_dense_2_y = nn.Linear(self.nb_dense, 1, bias=True)
        self.final_dense_1_r = nn.Linear(5184, self.nb_dense, bias=True)
        self.final_dense_2_r = nn.Linear(self.nb_dense, 1, bias=True)

    def forward(self, inputs, params=None):

        big = inputs[0]
        small = inputs[1]

        raw_input = torch.zeros_like(small)
        diff_input = small

        transform = T.Resize((9,9))
        for i in range(big.shape[0]):
            # iterate through batch
            raw_input[i,:,:,:] = transform(big[i,:,:,:])

        diff_input = self.TSM_1(diff_input)
        d1 = torch.tanh(self.motion_conv1(diff_input))
        d1 = self.TSM_2(d1)
        d2 = torch.tanh(self.motion_conv2(d1))

        r1 = torch.tanh(self.apperance_conv1(raw_input))
        r2 = torch.tanh(self.apperance_conv2(r1))

        g1 = torch.sigmoid(self.apperance_att_conv1(r2))
        g1 = self.attn_mask_1(g1)
        gated1 = d2 * g1

        # d3 = self.avg_pooling_1(gated1)
        # d4 = self.dropout_1(d3)

        # r3 = self.avg_pooling_2(r2)
        # r4 = self.dropout_2(r3)

        d4 = self.TSM_3(gated1)
        d5 = torch.tanh(self.motion_conv3(d4))
        d5 = self.TSM_4(d5)
        d6 = torch.tanh(self.motion_conv4(d5))

        r5 = torch.tanh(self.apperance_conv3(r2))
        r6 = torch.tanh(self.apperance_conv4(r5))

        g2 = torch.sigmoid(self.apperance_att_conv2(r6))
        g2 = self.attn_mask_2(g2)
        gated2 = d6 * g2

        # d7 = self.avg_pooling_3(gated2)
        # d8 = self.dropout_3(d7)
        d9 = gated2.view(gated2.size(0), -1)

        d10 = torch.tanh(self.final_dense_1_y(d9))
        d11 = self.dropout_4_y(d10)
        out_y = self.final_dense_2_y(d11)

        d10 = torch.tanh(self.final_dense_1_r(d9))
        d11 = self.dropout_4_r(d10)
        out_r = self.final_dense_2_r(d11)

        return out_y, out_r

#######################################################################################
####################################### DEEPPHYS  ######################################
#######################################################################################

class DeepPhys(nn.Module):

    def __init__(self, in_channels=3, nb_filters1=32, nb_filters2=64, kernel_size=3, dropout_rate1=0.25,
                 dropout_rate2=0.5, pool_size=(2, 2), nb_dense=128, out_size=1, img_size=36):
        """Definition of DeepPhys.
        Args:
          in_channels: the number of input channel. Default: 3
          img_size: height/width of each frame. Default: 36.
        Returns:
          DeepPhys model.
        """
        super(DeepPhys, self).__init__()

        print("INIT DEEPPHYS")

        self.in_channels = in_channels
        self.kernel_size = kernel_size
        self.dropout_rate1 = dropout_rate1
        self.dropout_rate2 = dropout_rate2
        self.pool_size = pool_size
        self.nb_filters1 = nb_filters1
        self.nb_filters2 = nb_filters2
        self.nb_dense = nb_dense
        self.out_size = out_size

        # Motion branch convs
        self.motion_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.motion_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        # Apperance branch convs
        self.apperance_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        self.apperance_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1), bias=True)
        # Attention layers
        self.apperance_att_conv1 = nn.Conv2d(self.nb_filters1, 1, kernel_size=1, padding=(0, 0), bias=True)
        self.attn_mask_1 = Attention_mask()
        self.apperance_att_conv2 = nn.Conv2d(self.nb_filters2, 1, kernel_size=1, padding=(0, 0), bias=True)
        self.attn_mask_2 = Attention_mask()
        # Dropout layers
        self.dropout_4 = nn.Dropout(self.dropout_rate2)
        # Dense layers
        self.final_dense_1 = nn.Linear(5184, self.nb_dense, bias=True)
        self.final_dense_2 = nn.Linear(self.nb_dense, self.out_size, bias=True)

    def forward(self, inputs, params=None):

        big = inputs[0]
        small = inputs[1]

        raw_input = torch.zeros_like(small)
        diff_input = small

        transform = T.Resize((9,9))
        for i in range(big.shape[0]):
            raw_input[i,:,:,:] = transform(big[i,:,:,:])

        d1 = torch.tanh(self.motion_conv1(diff_input))
        d2 = torch.tanh(self.motion_conv2(d1))

        r1 = torch.tanh(self.apperance_conv1(raw_input))
        r2 = torch.tanh(self.apperance_conv2(r1))

        g1 = torch.sigmoid(self.apperance_att_conv1(r2))
        g1 = self.attn_mask_1(g1)
        gated1 = d2 * g1

        d5 = torch.tanh(self.motion_conv3(gated1))
        d6 = torch.tanh(self.motion_conv4(d5))

        r5 = torch.tanh(self.apperance_conv3(r2))
        r6 = torch.tanh(self.apperance_conv4(r5))

        g2 = torch.sigmoid(self.apperance_att_conv2(r6))
        g2 = self.attn_mask_2(g2)
        gated2 = d6 * g2

        d9 = gated2.view(gated2.size(0), -1)
        d10 = torch.tanh(self.final_dense_1(d9))
        d11 = self.dropout_4(d10)
        out = self.final_dense_2(d11)

        return out


import json

# this will create a tweet, with possiblities of adding medias and replying to other tweets
def create_tweet(tas, message, media_ids = None, reply_ids = None):
	payload = {"status": message}

	if media_ids != None:
		payload["media_ids"] = media_ids

	if reply_ids != None:
		payload["in_reply_to_status_id"] = reply_ids

	r = tas.post("https://api.twitter.com/1.1/statuses/update.json", data = payload)
	resp = json.loads(r.text)

	if r.status_code == 200:
		tweet_id = resp["id"]
		return 0, (tweet_id,)

	return 1, (r.text,)

# this will delete a tweet based on a given tweet-id
def delete_tweet(tas, tweet_id):
	r = tas.delete(f"https://api.twitter.com/2/tweets/{tweet_id}")
	resp = json.loads(r.text)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct  5 16:56:55 2022

@author: josephbriggs
"""
import pathlib
import argparse
import cv2

def main():
    '''
    Converts files to greyscale.
    '''

    parser = argparse.ArgumentParser(description='Convert files to greyscale.')
    parser.add_argument('--input_path', "-i", type=str,
                        help='path to the image or directory of images. \
                If converting a directory, use *')
    parser.add_argument('--output_path', "-o", type=str,
                        help='output path where images will be saved.')
    parser.add_argument('--res', "-r", type=int,
                        help='downscale factor.')

    args = parser.parse_args()

    pathlib.Path(args.output_path).mkdir(parents=True, exist_ok=True)

    # files = pathlib.Path(args.input_path).glob(r'/*.png|')
    file_extentions = ['png', 'jpeg', 'jpg']
    files = []
    for file_extension in file_extentions:
        files += pathlib.Path(args.input_path).glob(fr'*.{file_extension}')

    for file in files:
        file_name = file.name
        image = cv2.imread(str(file))
        image_gs = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        file_name_to_save = args.output_path+"/"+file_name
        print(file_name_to_save)
        cv2.imwrite(file_name_to_save, image_gs)

    print('converted files to greyscale')

if __name__ == "__main__":

    main()


from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults

class DataQualityOperator(BaseOperator):
    """
    Runs data quality check by passing test SQL

    Parameters
    redshift_conn_id: Redshift Connection ID
    test_sql: SQL query to run on Redshift for data validation
    expected_result: Expected result to match the test result.

    """

    ui_color = '#89DA59'

    @apply_defaults
    def __init__(self,
                 redshift_conn_id="",
                 test_sql="",
                 expected_result="",
                 *args, **kwargs):

        super(DataQualityOperator, self).__init__(*args, **kwargs)
        self.redshift_conn_id=redshift_conn_id
        self.test_sql=test_sql
        self.expected_result=expected_result

    def execute(self, context):
        self.log.info("Start data validation...")
        redshift_hook = PostgresHook(Postgres_conn_in=self.redshift_conn_id)

        self.log.info("Got credentials.")
        records=redshift_hook.get_records(self.test_sql)

        if records[0][0] != self.expected_result:
            raise ValueError(f"Data quality check failed. {records[0][0]} does not equal {self.expected_result}")
        else:
            self.log.info("Data quality check passed!!!")


import numpy as np
import time
import matplotlib.pyplot as plt

a=np.loadtxt('meas2/magnitude_0to40.0mA_freq_sweep.csv', delimiter=',')
c=np.loadtxt('meas2/phase_0to40.0mA_freq_sweep.csv', delimiter=',')
b=np.loadtxt('meas2/sweep_feq.csv', delimiter=',')

cstart=0 #start current
cstop=40E-3 # stop current
cstep=5E-3 # current step
csteps=int((cstop-cstart)/cstep)

fig, (ax0,ax1)=plt.subplots(2,1, sharex=True)
for i in range(csteps):
	current=cstart+i*cstep
	ax0.plot(b,a[:,i], label="{0:d}mA".format(int((current)*1000)))
	ax1.plot(b,c[:,i], label="{0:d}mA".format(int((current)*1000)))

ax1.set_xlabel("Larmor frequency in kHz")
ax1.axhline(y=0, color='r', ls='--')
plt.legend(prop={'size':6})
#plt.savefig('meas2/phase_amplitude.png')
plt.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os.path
import argparse
import logging
from tarfile import TarFile

from thirdparty.dagflow import ParallelTask, Task, DAG, do_dag
from ontbc.common import mkdir, touch, read_tsv
from ontbc.parser import add_barcode_parser
from ontbc.config import PORECHOP_BIN, QUEUE
from ontbc import __file__, __version__, __email__, __author__

LOG = logging.getLogger(__name__)

def read_tar(file):

    a = os.path.dirname(file)

    return [os.path.join(a, i) for i in TarFile(file).getnames()]

def scan_cell(cell):

    fastqs = []
    summarys = []
    fast5s = []

    for root, dirs, files in os.walk(cell, followlinks=True, topdown=False):

        for name in files:
            path = os.path.join(root, name)
            if name.endswith(".fastq"):
                fastqs.append(path)
            elif name.endswith(".txt"):
                summarys.append(path)
            elif name.endswith(".fast5"):
                fast5s.append(path)
            elif name.endswith(".tar"):
                fast5s += read_tar(path)
            else:
                pass

    return fastqs, summarys, fast5s

def create_porechop_tasks(cell, barcodes, job_type, work_dir, out_dir):

    LOG.info("find fastq, summary and fast5 files in %r" % cell)

    fastq_fofn = os.path.join(work_dir, "fastq.fofn")
    summary_fofn = os.path.join(work_dir, "summary.fofn")
    fast5_fofn = os.path.join(work_dir, "fast5.fofn")
    find_done = os.path.join(work_dir, "find_done")

    if not os.path.exists(find_done):
        fastqs, summarys, fast5s = scan_cell(cell)

        for i, j in zip([fastq_fofn, summary_fofn, fast5_fofn], [fastqs, summarys, fast5s]):
            with open(i, "w") as fh:
                fh.write("%s\n" % "\n".join(j))

        del fastqs, summarys, fast5s
        touch(find_done)

    fastqs = [i[0] for i in read_tsv(fastq_fofn)]
    summarys = [i[0] for i in read_tsv(summary_fofn)]
    fast5s = [i[0] for i in read_tsv(fast5_fofn)]
    LOG.info("%s fastq, %s summary and %s fast5 files found" % (len(fastqs), len(summarys), len(fast5s)))

    del summarys, fast5s

    if job_type == "local":
        _option = ""
    else:
        _option = "-q %s" % ",".join(QUEUE)

    tasks = ParallelTask(
        id="bc",
        work_dir="%s/{id}" % work_dir,
        type=job_type,
        option=_option,
        script="""
{ontbc}/ontbc.py clean {{fastq}} > clean.fastq
{porechop}/porechop-runner.py -i clean.fastq -b . -t 1 --verbosity 2 --no_split > porechop.log
rm -rf clean.fastq
""".format(
            porechop=PORECHOP_BIN,
            ontbc=os.path.join(os.path.dirname(__file__), "..")
        ),
        fastq=fastqs,
    )

    summary = os.path.join(work_dir, "all.summary.txt")

    join_summary = Task(
        id="join_summary",
        work_dir=work_dir,
        type=job_type,
        script="""
less {summary} | xargs cat - > all.summary.txt
""".format(
            summary=summary_fofn
        ),
    )

    join_tasks = ParallelTask(
        id="join",
        work_dir=work_dir,
        type=job_type,
        script="""
mkdir -p {out}/{{barcode}}

if [ ! -e {{barcode}}_cat_done ]; then
    cat */{{barcode}}.fastq > {out}/{{barcode}}/{{barcode}}.fastq
    touch {{barcode}}_cat_done
fi

rm -rf */{{barcode}}.fastq

cd {out}/{{barcode}}
{ontbc}/ontbc.py filter --fastq {{barcode}}.fastq --summary {summary} --fast5 {fast5} \\
  --min_score -100 --min_length 0 --out {{barcode}}
rm {{barcode}}.filtered.fastq
mv {{barcode}}.filtered.summary.txt {{barcode}}.summary.txt
""".format(
            summary=summary,
            ontbc=os.path.join(os.path.dirname(__file__), ".."),
            fast5=fast5_fofn,
            out=out_dir
        ),
        barcode=barcodes
    )

    for i in join_tasks:
        i.set_upstream(*tasks)
        i.set_upstream(join_summary)

    return tasks, join_tasks, join_summary

def run_porechop(cell, barcodes, job_type, threads, work_dir, out_dir):

    assert os.path.isdir(cell), "%r not exist" % cell

    out_dir = mkdir(out_dir)
    work_dir = mkdir(work_dir)

    tasks, join_tasks, join_summary = create_porechop_tasks(
        cell=cell,
        barcodes=barcodes,
        job_type=job_type,
        work_dir=work_dir,
        out_dir=out_dir
    )

    dag = DAG("porechop")

    dag.add_task(*tasks)
    dag.add_task(join_summary)
    dag.add_task(*join_tasks)

    do_dag(dag, concurrent_tasks=threads, refresh_time=30)

def barcode(args):

    run_porechop(
        cell=args.cell,
        barcodes=args.barcode,
        job_type=args.job_type,
        threads=args.threads,
        work_dir=args.work_dir,
        out_dir=args.out_dir
    )

def main():
    logging.basicConfig(
        stream=sys.stderr,
        level=logging.INFO,
        format="[%(levelname)s] %(message)s"
    )

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""

version: %s
contact:  %s <%s>\
    """ % (__version__, " ".join(__author__), __email__))

    parser = add_barcode_parser(parser)
    args = parser.parse_args()
    barcode(args)

if __name__ == "__main__":
    main()


from IPython import get_ipython

def type_of_script():
    """
    Detects and returns the type of python kernel
    :return: string 'jupyter' or 'ipython' or 'terminal'
    """
    try:
        ipy_str = str(type(get_ipython()))
        if 'zmqshell' in ipy_str:
            return 'jupyter'
        if 'terminal' in ipy_str:
            return 'ipython'
    except:
        return 'terminal'

if type_of_script() == 'jupyter':
    from tqdm.notebook import tqdm
else:
    from tqdm import tqdm

import matplotlib.pyplot as plt # type: module
import matplotlib.ticker as ticker
from matplotlib import colormaps
from matplotlib.colors import Normalize
import matplotlib.gridspec as gridspec

import numpy as np
import os, glob
import time
import warnings

from rur.fortranfile import FortranFile
from rur import uri, uhmi, painter, drawer
from rur.sci.photometry import measure_luminosity
from rur.sci.geometry import get_angles, euler_angle
from rur.utool import rotate_data
from scipy.ndimage import gaussian_filter
uri.timer.verbose=1
# from rur.sci.kinematics import f_getpot

from icl_IO import mode2repo, pklsave, pklload
from icl_tool import *
from icl_numba import large_isin, large_isind, isin
from icl_draw import drawsnap, add_scalebar, addtext, MakeSub_nolabel, label_to_in, fancy_axis, circle
import argparse, subprocess
from importlib import reload
import cmasher as cmr
from copy import deepcopy
from multiprocessing import Pool, shared_memory

mode = 'nh'
iout = 1026
repo, rurmode, dp = mode2repo(mode)
snap = uri.RamsesSnapshot(repo, iout, mode=rurmode)
snaps = uri.TimeSeries(snap)
snaps.read_iout_avail()
nout = snaps.iout_avail['iout']
gals = uhmi.HaloMaker.load(snap, galaxy=True, double_precision=dp)
hals = uhmi.HaloMaker.load(snap, galaxy=False, double_precision=dp)
database = f"/home/jeon/MissingSat/database"

from common_func import *

tree = pklload(f"{database}/02_main_progenitors.pickle")
if(os.path.exists(f"{database}/halo_dict.pickle")):
    halos = pklload(f"{database}/halo_dict.pickle")
else:
    halos = {'catalog':{}, 'index':{}}
    uri.timer.verbose=0
    for iout in tqdm(np.unique(tree['timestep'])):
        isnap = snaps.get_snap(iout)
        ihals = uhmi.HaloMaker.load(isnap, galaxy=False, double_precision=dp)
        indicies = np.zeros(len(ihals), dtype=int)
        iids = tree[tree['timestep'] == iout]['id']
        ihals = ihals[iids-1]
        indicies[iids-1] = np.arange(len(iids))
        halos['catalog'][iout] = ihals
        halos['index'][iout] = indicies
    pklsave(halos, f"{database}/halo_dict.pickle")

def _ibox(h, factor=1):
    return np.array([
                    [h['x']-factor*h['r'], h['x']+factor*h['r']],
                    [h['y']-factor*h['r'], h['y']+factor*h['r']],
                    [h['z']-factor*h['r'], h['z']+factor*h['r']]
                    ])

uri.timer.verbose=0
for iout in np.unique(tree['timestep'])[::-1]:
    if(os.path.exists(f"{database}/main_prog/cpulist/cpulist_{iout:05d}.pickle")): continue
    cpudict = {}
    targets = halos['catalog'][iout]
    isnap = snaps.get_snap(iout)

    cpulists = []
    with Pool(32) as pool:
        async_result = [
                    pool.apply_async(
                            uri.get_cpulist,
                            (_ibox(h,factor=1.1), None, isnap.levelmax, isnap.bound_key, isnap.ndim, 5, isnap.ncpu)
                            ) for h in targets
                    ]
        iterobj = tqdm(async_result, total=len(targets), desc=f"iout={iout:05d}")
        for r in iterobj:
            cpulists.append(r.get())
    cpulists = np.unique( np.concatenate(cpulists) )
    cpudict['all'] = cpulists
    pklsave(cpudict, f"{database}/main_prog/cpulist/cpulist_{iout:05d}.pickle")
    print(f"`{database}/main_prog/cpulist/cpulist_{iout:05d}.pickle` save done")
    isnap.clear()


#
# db.py
#
import os
import sqlite3
import time
import datetime
from flask import g
import db_init

def test_db_conn():
    dbname = os.environ.get("SEVENS_DB_NAME")

    try:
        os.remove(dbname)
    except OSError:
        pass
    """ Check if db connection can open, create and init db if doesn't already exist """
    print("********dbname: "+dbname)

    assert dbname is not None

    # If db file doesn't exist call the initialize module to create and init
    if not os.path.isfile(dbname):
        print("!!!!!!!!!dbname: "+dbname)
        db_init.init(dbname)

def get_db():
    dbname = os.environ.get("SEVENS_DB_NAME")
    assert dbname is not None

    if not hasattr(g,'sqlite_db'):
        g.sqlite_db = sqlite3.connect(dbname)
    return g.sqlite_db

def close_db():
    if hasattr(g,'sqlite_db'):
	    g.sqlite_db.close()

#     CREATE TABLE players (id INTEGER PRIMARY KEY AUTOINCREMENT, points INTEGER, name STRING);
def players_row_to_dict(row):
    players = {}
    players['id'] = row[0]
    players['points'] = row[1]
    players['name'] = row[2]
    return players

#CREATE TABLE hands (player_id INTEGER FOREIGN KEY, clubs STRING, hearts STRING, diamonds STRING, spades STRING);
def hands_row_to_dict(row):
    hands = {}
    hands['player_id'] = row[0]
    hands['clubs'] = str(row[1])
    hands['hearts'] = str(row[2])
    hands['diamonds'] = str(row[3])
    hands['spades'] = str(row[4])
    return hands

# CREATE TABLE board (cur_player_id INTEGER FOREIGN KEY, clubs STRING, hearts STRING, diamonds STRING, spades STRING);
def board_row_to_dict(row):
    board = {}
    board['cur_player_id'] = row[0]
    board['clubs'] = str(row[1])
    board['hearts'] = str(row[2])
    board['diamonds'] = str(row[3])
    board['spades'] = str(row[4])
    return board

def get_game_state():
    conn = get_db()
    curs = conn.cursor()
    rows = curs.execute ("SELECT * FROM board").fetchall()
    board = []
    for row in rows:
        b = board_row_to_dict(row)
        board.append(b)
    return board

def get_games():
    conn = get_db()
    curs = conn.cursor()
    rows = curs.execute ("SELECT * FROM games ORDER BY date").fetchall()
    games = []
    for row in rows:
        game = game_row_to_dict(row)
        games.append(game)
    return games

'''
def update_game(gameid,score,lines,user):
    conn = get_db()
    curs = conn.cursor()
    curs.execute("UPDATE games SET score=?, lines=?, user=?, haveResult=? WHERE id=?;",
                (score, lines, user, True, gameid))
    conn.commit()
    res = curs.execute("SELECT * FROM games WHERE id=?;",(gameid,)).fetchall()
    if len(res) != 0:
       return game_row_to_dict(res[0])
    else:
       return None

def add_access_log(game_id, func, method, auth, ip, user_agent):
    """ Add access log to global access_log list """
    conn = get_db()
    curs = conn.cursor()

    curs.execute("INSERT INTO accesslog (id, function, method, date, ipaddress, useragent, user) VALUES (?,?,?,?,?,?,?);", (game_id, func, method, time.time(),ip, user_agent, auth))
    conn.commit()

def get_access_logs():
    conn = get_db()
    curs = conn.cursor()
    rows = curs.execute ("SELECT * FROM accesslog ORDER BY date").fetchall()
    access_log = []
    for row in rows:
        access = accesslog_row_to_dict(row)
        access_log.append(access)
    return access_log

def get_rng_seed():
    """ Generate rng seed """
    return 0xDEADBEEF
'''


#!/usr/bin/env python
#
# fast_mr ->
#
# Fast molecular replacement in the spirit of fast_dp, starting from coordinate
# files and using brute force (and educated guesses) to get everything going.
#
# fast_mr - main program.

import os
import sys
import time
import shutil
import math
import traceback
from multiprocessing import Pool

from iotbx import mtz
from iotbx import pdb
from libtbx.phil import parse
from cctbx.sgtbx import space_group, space_group_symbols
from iotbx.scalepack import merge as merge_scalepack
from libtbx import introspection

if not 'FAST_EP_ROOT' in os.environ:
    raise RuntimeError('FAST_EP_ROOT not set')

fast_ep_lib = os.path.join(os.environ['FAST_EP_ROOT'], 'lib')

if not fast_ep_lib in sys.path:
    sys.path.append(fast_ep_lib)

from xml_output import write_ispyb_xml

from generate_possible_spacegroups import generate_chiral_spacegroups, \
     spacegroup_enantiomorph, spacegroup_full, sanitize_spacegroup
from run_job import run_job, run_job_cluster, is_cluster_job_finished
from fast_mr_phaser import run_phaser_cluster
from parse_pdb import pdb_file_nres

class logger:
    def __init__(self):
        self._fout = open('fast_mr.log', 'w')
        return

    def __del__(self):
        self._fout.close()
        self._cout = None
        return

    def __call__(self, _line):
        sys.stdout.write('%s\n' % _line)
        self._fout.write('%s\n' % _line)
        return

class Fast_mr:

    def __init__(self, hklin, xyzin_and_ids):

        self._hklin = os.path.abspath(hklin)
        self._xyzins = [os.path.abspath(xyzin_and_id[0])
                        for xyzin_and_id in xyzin_and_ids]
        self._ids = [xyzin_and_id[1] for xyzin_and_id in xyzin_and_ids]
        self._cpu = 2
        self._machines = 10
        self._wd = os.getcwd()
        self._log = logger()

        self._log('Using %d cpus / %d machines' % (self._cpu, self._machines))

        self._full_command_line = ' '.join(sys.argv)

        # pull information we'll need from the input MTZ file - the unit cell,
        # the pointgroup and the number of reflections in the file. select
        # first Miller array in file which has native data

        # --- SAD DATA ---

        m = mtz.object(self._hklin)
        mas = m.as_miller_arrays()

        self._data = None

        for ma in mas:
            if str(ma.observation_type()) != 'xray.amplitude':
                continue
            self._data = ma
            break

        if not self._data:
            raise RuntimeError('no intensity data found in %s' % \
                self._hklin)

        self._pointgroup = self._data.space_group().type().number()
        self._unit_cell = self._data.unit_cell().parameters()

        self._nrefl = m.n_reflections()
        self._spacegroups = generate_chiral_spacegroups(self._pointgroup)

        # write out a nice summary of the data set properties and what columns
        # were selected for analysis

        self._log('Input:       %s' % self._hklin)
        self._log('Columns:     %s' % self._data.info().label_string())
        self._log('Unit cell:   %.2f %.2f %.2f %.2f %.2f %.2f' % \
                  self._unit_cell)
        self._log('Pointgroup:  %s' % m.space_group().type().lookup_symbol())
        self._log('Resolution:  %.2f - %.2f' % self._data.resolution_range())
        self._log('Nrefl:       %d' % self._nrefl)
        self._log('Spacegroups: %s' % ' '.join(self._spacegroups))

        self._log('Input coordinate files:')
        self._nres = []
        for xyzin, _id in zip(self._xyzins, self._ids):
            nres = pdb_file_nres(xyzin)
            self._nres.append(nres)
            self._log('%40s %8d %.3f' % (os.path.split(xyzin)[1], nres, _id))

        total_nres = sum(self._nres)
        # FIXME calculate probable number of complexes in here

        self._copies = 1

        return

    def do_mr(self):
        t0 = time.time()

        cluster = True
        njobs = self._machines
        ncpu = self._cpu

        # set up N phaser jobs

        jobs = [ ]

        for spacegroup in self._spacegroups:
            wd = os.path.join(self._wd, spacegroup)
            if not os.path.exists(wd):
                os.makedirs(wd)
            commands = ['mode mr_auto',
                        'spacegroup %s' % spacegroup,
                        'hklin %s' % self._hklin,
                        'labin F=F SIGF=SIGF',
                        'root mr%s' % spacegroup]
            for j, (xyzin, _id, nres) in enumerate(
                zip(self._xyzins, self._ids, self._nres)):
                commands.append('ensemble m%d pdb %s identity %f' %
                                (j, xyzin, 100 * _id))
            for j, (xyzin, _id, nres) in enumerate(
                zip(self._xyzins, self._ids, self._nres)):
                commands.append('composition protein nres %d num %d' %
                                (nres, self._copies))
            for j, (xyzin, _id, nres) in enumerate(
                zip(self._xyzins, self._ids, self._nres)):
                commands.append('search ensemble m%d num %d' %
                                (j, self._copies))

            jobs.append((wd, commands))

        # actually execute the tasks - either locally or on a cluster, allowing
        # for potential for fewer available machines than jobs

        self._log('Running %d x phaser jobs' % len(jobs))

        pool = Pool(min(njobs, len(jobs)))

        if cluster:
            pool.map(run_phaser_cluster, jobs)
        else:
            print(1/0)

        # now look for the results
        worked = []
        for job in jobs:
            wd = job[0]
            spacegroup = os.path.split(wd)[-1]
            if os.path.exists(os.path.join(wd, 'mr%s.sol' % spacegroup)):
                worked.append(os.path.join(wd, 'mr%s.sol' % spacegroup))

        for w in worked:
            sol = open(w).read()
            for record in sol.split('\n'):
                if 'SOLU SPAC' in record:
                    spacegroup = record.replace(
                        'SOLU SPAC', '').replace(' ', '')
                if 'SOLU SET' in record:
                    tfz = float(record.replace('=', ' ').split()[5])
            print('Solution: %s %.2f' % (spacegroup, tfz))

        t1 = time.time()
        self._log('Time: %.2f' % (t1 - t0))

if __name__ == '__main__':
    xyzin_and_ids = []
    for arg in sys.argv[2:]:
        if ':' in arg:
            xyzin = arg.split(':')[0]
            _id = float(arg.split(':')[1])
            if _id > 1.0:
                _id /= 100.0
            xyzin_and_ids.append((xyzin, _id))
        else:
            xyzin_and_ids.append((arg, 1.0))

    fast_mr = Fast_mr(sys.argv[1], xyzin_and_ids)
    try:
        fast_mr.do_mr()
    except RuntimeError as e:
        fast_mr._log('*** MR: %s ***' % str(e))
        traceback.print_exc(file = open('fast_mr.error', 'w'))
        sys.exit(1)


import os
import cv2
import numpy as np
import shutil
import sys
sys.path.insert(0,os.path.realpath('..'))
sys.path.insert(0,os.path.join(os.path.realpath('..'),'piano_utils'))
from tools.warper import order_points
from config import cfg
from piano_utils.networks import PSPNet
from piano_utils.util import colorize_mask
from piano_utils.keyboard import KeyBoard
from PIL import Image
from tqdm import tqdm
import shapely
from shapely.geometry import Polygon,MultiPoint
import time
from skimage.measure import label, regionprops
from collections import Counter
import json
from IPython import embed
import pickle

exp_cfg = {
        'exp_imgs':'/home/data/lj/Piano/experment/keyboard/exp_imgs',
        'tmp_dir':'/home/data/lj/Piano/experment/keyboard/tmp_dir',
        'figure_dir':'/home/data/lj/Piano/experment/keyboard_figure'
}
class HoughKeyBoard(object):
    def __init__(self):
        self.theta_thersh = 0.08

    def hough_transform(self,img):
        res = {}
        img_ori = img.copy()
        h, w = img.shape[:2]
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        gray = cv2.GaussianBlur(gray, (5, 5), 0)
        edges = cv2.Canny(gray, 50, 150, 3)
        lines = cv2.HoughLines(edges, 1.0, np.pi / 180, 120)
        thetas = [x[0][1] for x in lines if not (x[0][1] < (np.pi / 4.) or
                  x[0][1] > (3.*np.pi/4.0))]
        dic = dict(Counter(thetas))
        theta = sorted(dic.items(), key=lambda x: x[1], reverse=True)
        if len(theta) > 0 and theta[0][1] > 1:   #---统计角度最多重复的直线
            most_theta = theta[0][0]
        else:
            return
        x_loc, y_loc, pts = [], [], []
        for line in lines:
            rho, theta = line[0]
            if abs(most_theta * 180 / np.pi - 90) > 1.5:  #--键盘是斜着的
                if abs(theta - most_theta) > self.theta_thersh:
                    continue
            else:    #---其他情况
                if not theta == most_theta:
                    continue
            pt1 = (0, max(int(rho / np.sin(theta)), 0))
            pt2 = (img_ori.shape[1], max(int((rho - img_ori.shape[1] * np.cos(theta)) / np.sin(theta)),0))
            cv2.line(img_ori, pt1, pt2, (0, 255, 0), 1)
            pts.append((pt1, pt2))
        return img_ori

def get_img_box_dict():
    img_box_dict = dict()
    file_name = '/home/data/lj/Piano/Segment/train.txt'
    with open(file_name,'r') as fr:
        items = [l.strip() for l in fr.readlines()]
    mask_lists = []
    for item in items:
        item = item.split()
        if 'tools' in item[0]:
            #mask_dir = item[0].split('/')[-2]
            continue
        else:
            mask_dir = os.path.basename(item[0]).split('_img_') [0]
        if 'segment' in item[0]:continue
        if mask_dir in mask_lists:continue
        mask_lists.append(mask_dir)
        img_mask = cv2.imread(item[1],cv2.IMREAD_GRAYSCALE)
        img_mask[img_mask==2] = 1
        img_mask[img_mask==1] = 255
        contours,_ = cv2.findContours(img_mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
        assert len(contours)==1,'value wrong'
        contours = np.squeeze(contours)
        rect = order_points(contours).reshape(-1,1,2).astype(int)
        img_box_dict[mask_dir] = rect

    json_path = os.path.join(exp_cfg['exp_imgs'],'need_labels')
    json_files = [os.path.join(json_path,x) for x in os.listdir(json_path) if x.endswith('json')]
    json_files.sort()
    for json_file in json_files:
        with open(json_file,'r') as fr:
            items = json.load(fr)
        basename = os.path.basename(json_file).split('.')[0]
        points = np.array(items['shapes'][0]['points'])
        rect = order_points(points).reshape(-1,1,2).astype(int)
        img_box_dict[basename] = rect
    return img_box_dict

class KeyBoard_Exp(KeyBoard):
    def __init__(self):
        KeyBoard.__init__(self)
        print('KeyBoard load finish')

    def detect(self,img):
        image = img.convert('RGB')
        self.prediction = self.inference(img)
        contours,_ = self.find_contours(image,self.prediction)
        rect = order_points(contours).reshape(-1,1,2).astype(int)
        return rect

    def mask2image(self,image):
        image = Image.fromarray(cv2.cvtColor(image,cv2.COLOR_BGR2RGB))
        w, h = image.size
        colorized_mask = colorize_mask(self.prediction, self.palette)
        output_im = Image.new('RGB', (w*2, h))
        output_im.paste(image, (0,0))
        output_im.paste(colorized_mask, (w,0))
        output_im = cv2.cvtColor(np.asarray(output_im),cv2.COLOR_RGB2BGR)
        return output_im

def cal_iou(gt_rect, det_rect):
    #---不规则的两个四边形计算Iou,不是矩形了
    gt_rect = gt_rect.reshape(4, 2)
    poly1 = Polygon(gt_rect).convex_hull
    det_rect = det_rect.reshape(4,2)
    poly2 = Polygon(det_rect).convex_hull
    union_poly = np.concatenate((gt_rect,det_rect))
    if not poly1.intersects(poly2):
        iou = 0
    else:
        try:
            inter_area = poly1.intersection(poly2).area
            union_area = MultiPoint(union_poly).convex_hull.area
            if union_area == 0:
                iou= 0
            iou=float(inter_area) / union_area
        except shapely.geos.TopologicalError:
            print('shapely.geos.TopologicalError occured, iou set to 0')
            iou = 0
    return iou

def ensure_dir(path):
    if not os.path.exists(path):
        os.makedirs(path)

def main():
    seg_pickle_file = os.path.join(exp_cfg['tmp_dir'],'seg.pkl')
    hour_pickle_file = os.path.join(exp_cfg['tmp_dir'],'hourgh.pkl')
    path = exp_cfg['exp_imgs']
    save_seg_dir = os.path.join(exp_cfg['figure_dir'],'segment')
    save_hourgh_dir = os.path.join(exp_cfg['figure_dir'],'hourgh')
    ensure_dir(save_seg_dir)
    ensure_dir(save_hourgh_dir)

    img_files = [os.path.join(path,x) for x in os.listdir(path)]
    gt_box_dict = get_img_box_dict()

    with open(seg_pickle_file,'rb') as f1:
        seg_box_dict = pickle.load(f1)
    with open(hour_pickle_file,'rb') as f2:
        hour_box_dict = pickle.load(f2)

    seg_ious = []
    for img_mark,det_rect in seg_box_dict.items():
        gt_rect = gt_box_dict[img_mark]
        iou = cal_iou(gt_rect,det_rect)
        if iou>0.5:
            seg_ious.append(iou)
        else:print(img_mark)

    hour_detector = HoughKeyBoard()
    keyboard_net = KeyBoard_Exp()

    hour_ious = []
    for img_mark,det_rect in hour_box_dict.items():
        gt_rect = gt_box_dict[img_mark]
        iou = cal_iou(gt_rect,det_rect)
        if iou>0.5:
            hour_ious.append(iou)
        else:
            img = cv2.imread(os.path.join(path,img_mark+'.jpg'))
            img_copy = img.copy()
            img_input = Image.fromarray(cv2.cvtColor(img_copy,cv2.COLOR_BGR2RGB))
            seg_rect = keyboard_net.detect(img_input)
            for rect in det_rect:
                rect = rect[0]
                cv2.circle(img,(rect[0],rect[1]),5,(0,255,0),3)
            for rect in seg_rect:
                rect = rect[0]
                cv2.circle(img_copy,(rect[0],rect[1]),5,(0,255,0),3)
            img_copy = keyboard_net.mask2image(img_copy)
            img = hour_detector.hough_transform(img)
            cv2.imwrite(os.path.join(save_hourgh_dir,img_mark+'.jpg'),img)
            cv2.imwrite(os.path.join(save_seg_dir,img_mark+'.jpg'),img_copy)

if __name__=='__main__':
    main()


from .base import metadata

from sqlalchemy import Table, Column, BigInteger,\
    String, Boolean, DateTime

t_users = Table(
    "users",
    metadata,
    Column('u_id',           BigInteger),  # telegram id
    Column('name',           String),  # фамилия c инициалами
    Column('name_tg',        String),  # имя пользователя в телеге если есть
    Column('admin',          Boolean),  # является ли администратором
    Column('org_name',       String),
    Column('org_code',       String),
    Column('date_update',    DateTime),
        )


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 20 09:17:19 2019

@author: if715029
"""

import numpy as np
import matplotlib.pyplot as plt
import scipy.spatial.distance as sc
import pandas as pd

#%%
data = pd.read_excel('../data/Datos_2015.xlsx',sheet_name='Atemajac')

#%%
data = data.iloc[:,2:7].dropna()

#%%
D1 = sc.squareform(sc.pdist(data.iloc[:,2:],'euclidean'))

#%%
data_norm = (data-data.mean(axis=0))/data.std(axis=0)

#%%
plt.subplot(1,2,1)
plt.scatter(data['CO'],data['PM10'])
plt.axis('square')
plt.subplot(1,2,2)
plt.scatter(data_norm['CO'],data_norm['PM10'])
plt.axis('square')
plt.show()


import models
import typing
import sqlite3
import os
import sys

class Storage:
    def __init__(self):
        self._conn = sqlite3.connect('v_store.db')
        self._cursor = self._conn.cursor()
        self._queries: dict[str, str] = self.read_queries()

    def __del__(self):
        self._cursor.close()
        self._conn.close()

    def read_queries(self) -> dict[str, str]:
        queries = {}
        current_key = None
        current_query = []
        with open("./queries.sql", "r") as f:
            for line in f:
                line = line.strip()
                if line.startswith("--"):
                    if current_key is not None:
                        queries[current_key] = "\n".join(current_query)
                        current_query = []
                    current_key = line[2:].strip()
                else:
                    current_query.append(line)
        if current_key is not None:
            queries[current_key] = "\n".join(current_query)
        return queries

    def make_table(self) -> int:
        try:
            self._cursor.execute(self._queries["make_vector_table"])
            return 0
        except Exception as e:
            raise e

    def add_point(self, point: models.Point) -> int:
        try:
            query = self._queries["add_point"].format(str(point), repr(point))
            self._cursor.execute(query)
            return 0
        except Exception as e:
            raise e

    def get_point(self, id) -> int | models.Vector:
        try:
            query = self._queries["get_point_by_id"].format(id)
            self._cursor.execute(query)
            vec: models.Vector =  self._cursor.fetchone()
            if vec:
                return exec(vec)
            else:
                raise ValueError("Point with that id does not exist")
        except Exception as e:
            raise e

    def delete_point(self, id) -> int:
        try:
            query = self._queries["delete_point_by_id"].format(id)
            self._cursor.execute(query)
            return 0
        except Exception as e:
            raise e

s = Storage()
print(s.read_queries())


#!/usr/bin/env python2.7

"""
    A tool to update the Product Version and Code of a C# VS2010 setup package (*.vdproj).  Intended
    to be used with an automated build process.

"""

import re
import uuid
import argparse
import os, shutil
import tempfile

##"ProductCode" = "8:{35424778-8534-431B-9492-5CD84B1EDE03}"
productcode_re = re.compile(r"(?:\"ProductCode\" = \"8.){([\d\w-]+)}")

##"ProductVersion" = "8:1.0.89"
productversion_re = re.compile(r"(?:\"ProductVersion\" = \"8.)([\d\w\.]+)\"")

def replace_code_and_version(src_fname, version="1.0.0", code="12345678-1234-1234-1234-1233456789012"):
    fd, tmp_fname = tempfile.mkstemp()
    tmp = open(tmp_fname, 'w')
    src = open(src_fname)
    for l in src:
        if productcode_re.search(l):
            m = productcode_re.search(l)
            l = l.replace(m.group(1), code)
        if productversion_re.search(l):
            m = productversion_re.search(l)
            l = l.replace(m.group(1), version)
        tmp.write(l)
    tmp.close()
    os.close(fd)
    src.close()
    os.remove(src_fname)
    shutil.move(tmp_fname, src_fname)

def parse_commands(test_args=None):
    descrip = "Utility to update ProductCode and ProductVersion of VS2010 setup projects"
    parser = argparse.ArgumentParser(description=descrip)
    parser.add_argument("-f", "--file", dest="vdproj", action="store", default=None,
            help="The vdproj file to be 'adjusted'", required=True)
    parser.add_argument("-v", "--version", action="store", dest="version", default="1.0.0",
            help="The new version to be set conforming to: major, minor, build e.g '1.0.195'")
    parser.add_argument("-c", "--code", dest="code", action="store", default=str(uuid.uuid4()),
            help="The new product code GUID. If not provided one is generated. ")

    ## Don't update the UpgradeCode that needs to stay the same for the product duration

    if test_args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(test_args)
    return args

def main():
    args = parse_commands()
    replace_code_and_version(args.file, args.version, args.code)

if __name__ == '__main__':
    main()


import os
import click

from guardata.utils import trio_run
from guardata.api.protocol import OrganizationID
from guardata.logging import configure_logging
from guardata.cli_utils import spinner, cli_exception_handler
from guardata.client.types import BackendAddr, BackendOrganizationBootstrapAddr
from guardata.client.backend_connection import apiv1_backend_administration_cmds_factory

async def _create_organization(debug, name, backend_addr, administration_token, expiration_date):
    async with spinner("Creating group in backend"):
        async with apiv1_backend_administration_cmds_factory(
            backend_addr, administration_token
        ) as cmds:
            rep = await cmds.organization_create(name, expiration_date)
            if rep["status"] != "ok":
                raise RuntimeError(f"Backend refused to create group: {rep}")
            bootstrap_token = rep["bootstrap_token"]

    organization_addr = BackendOrganizationBootstrapAddr.build(backend_addr, name, bootstrap_token)
    organization_addr_display = click.style(organization_addr.to_url(), fg="yellow")
    click.echo(f"Bootstrap group url: {organization_addr_display}")

@click.command(short_help="create new group")
@click.argument("name", required=True, type=OrganizationID)
@click.option("--addr", "-B", required=True, type=BackendAddr.from_url)
@click.option("--administration-token", "-T", required=True)
@click.option("--expiration-date", "-E", default=None, type=click.DateTime())
def create_organization(name, addr, administration_token, expiration_date):
    debug = "DEBUG" in os.environ
    configure_logging(log_level="DEBUG" if debug else "WARNING")
    with cli_exception_handler(debug):
        trio_run(_create_organization, debug, name, addr, administration_token, expiration_date)