import os.path
import math
import json
from typing import List, Optional

# 3rd
import ROOT

# ours
import metaoptions
import validationcomparison
import validationpath
from validationfunctions import strip_ext, index_from_revision, get_style
import json_objects
from validationrootobject import RootObject

# todo: [Ref, low prio, medium work] Refactor into class with uniform interface
#   and subclasses implementing actual functionality for Plot/Tuple etc.
#   /klieret
class Plotuple:

    """!
    A Plotuple is either a Plot or an N-Tuple

    @var work_folder: the work folder containing the results and plots
    @var root_objects: A list of Root-objects which belong
        together (i.e. should be drawn into one histogram or one table)
    @var revisions: The list of revisions
    @var warnings: A list of warnings that occured while creating the
        plots/tables for this Plotuple object
    @var reference: The reference RootObject for this Plotuple
    @var elements: The elements (RootObject of different revisions) for this
        Plotuple
    @var newest: The newest element in elements
    @var key: The key of the object within the corresponding ROOT file
    @var type: The type of the elements (TH1, TH2, TNtuple)
    @var description: The description of this Plotuple object
    @var check: Hint how the Plotuple object should look like
    @var contact: The contact person for this Plotuple object
    @var package: The package to which this Plotuple object belongs to
    @var rootfile: The rootfile to which the Plotuple object belongs to
    @var chi2test_result: The result of the Chi^2-Test. By default, there is no
        such result. If the Chi^2-Test has been performed, this variable holds
        the information between which objects it has been performed.
    @var pvalue: The p-value that the Chi^2-Test returned
    @var file: The file, in which the histogram or the HMTL-table (for
        n-tuples) are stored (without the file extension!)
    """

    def __init__(
        self,
        root_objects: List[RootObject],
        revisions: List[str],
        work_folder: str,
    ):
        """!
        The default constructor for a Plotuple-object.
        @param root_objects: A list of Root-objects which belong
            together (i.e. should be drawn into one histogram or one table)
        @param revisions: The list of revisions (Duh!)
        """

        # the work folder containing the results and plots
        self._work_folder = work_folder

        # The list of Root objects in this Plotuple-object
        self._root_objects = root_objects

        # The list of revisions
        self._revisions = revisions

        # A list of all problems that occured with this Plotuple,
        # e.g. missing reference object, missing meta-information...
        self.warnings: List[str] = []

        # Find the reference element. If we can't find one, set it to 'None'
        # The reference-object for this Plotuple object
        self._reference: Optional[RootObject] = None
        for root_object in self._root_objects:
            if root_object.is_reference:
                self._reference = root_object
                break

        # If we couldn't find a reference element, add that to warnings
        if self._reference is None:
            self.warnings = ["No reference object"]

        # All elements of the Plotuple that are not the reference-object
        # Get the elements, i.e. all RootObjects except for the
        # reference object. May be either histograms or n-tuples.
        # Note that the reference doesn't have a date set (and if we only plot
        # the reference, then is_reference is probably not set), so we have
        # to be careful of how to sort
        self._elements = [ro for ro in root_objects if not ro.is_reference]
        self._elements.sort(
            key=lambda ro: ro.date if ro.date else 0, reverse=True
        )

        # The newest element, i.e. the element belonging the revision
        # whose data were created most recently.
        # Should always be self.element[0], except if there is only a
        # reference object
        if self._elements:
            self._newest = self._elements[0]
        else:
            self._newest = self._reference

        # All available meta-information about the plotuple object:

        # The key (more precisely: the name of the key) that all elements of
        # this Plotuple object share
        self.key = self._newest.key

        # The type of the elements in this Plotuple object
        self.type = self._newest.type

        if self.type == "TNamed":
            # Sometimes, we use TNamed to encode extra information about the
            # ROOT file. In order to avoid that this will be plotted, we
            # catch it here and assign it the type 'Meta'
            meta_fields = ["description"]
            if self._newest.object.GetName().lower().strip() in meta_fields:
                self.type = "meta"

        # The description of the histogram/n-tuple which this Plotuple object
        # will yield
        self._description = self._newest.description

        # The 'Check for ...'-guideline for the histogram/n-tuple which this
        # Plotuple object will yield
        self._check = self._newest.check

        # A contact person for the histogram/n-tuple which this Plotuple object
        # will yield
        self._contact = self._newest.contact

        # MetaOptionParser for the meta-options for this Plotuple object
        self._mop = metaoptions.MetaOptionParser(self._newest.metaoptions)

        # The package to which the elements in this Plotuple object belong
        self.package = self._newest.package

        # The root file to which the elements in this Plotuple object belong
        self.rootfile = self._newest.rootfile

        # The result of the Chi^2-Test. By default, there is no such result.
        # If the Chi^2-Test has been performed, this variable holds between
        # which objects it has been performed.
        self._comparison_result_long = "n/a"

        self.comparison_result = "not_compared"

        # The json file, in which the ntuple information is stored
        self._file: Optional[str] = None

        self._html_content: Optional[str] = None

        #: width of the plotted image in pixels, will be set by the draw
        #: function
        self._width: Optional[int] = None

        #: height of the plotted image in pixels, will be set by the draw
        #: function
        self._height: Optional[int] = None

        # Deal with incomplete information
        if self._description == "" or self._description is None:
            self._description = "n/a"
            self.warnings.append("No description")
        if self._check == "" or self._check is None:
            self._check = "n/a"
            self.warnings.append("No Check")
        if self._contact == "" or self._contact is None:
            self._contact = "n/a"
            self.warnings.append("No Contact Person")

        #: The folder in which the plots will be found. Note that this should
        #: be relative to the ``html`` directory, because this is the string
        #: that will be used for the JavaScript queries to get the files
        self._plot_folder = os.path.join(
            validationpath.get_html_plots_tag_comparison_folder(
                work_folder, tags=revisions
            ),
            self.package,
        )
        os.makedirs(self._plot_folder, exist_ok=True)

    def has_reference(self):
        """!
        @return True if a reference file is found for this plotuple
        """
        return self._reference is not None

    def create_plotuple(self):
        """!
        Creates the histogram/table/image that belongs to this Plotuble-object.
        """

        if self.type == "TH1" or self.type == "TEfficiency":
            self.create_histogram_plot("1D")
        elif self.type == "TGraph":
            self.create_graph_plot()
        elif self.type == "TH2":
            self.create_histogram_plot("2D")
        # used to store HTML user content
        elif self.type == "TNamed":
            self.create_html_content()
        elif self.type == "TASImage":
            self.create_image_plot()
        elif self.type == "TNtuple":
            self.create_ntuple_table_json()
        elif self.type == "meta":
            pass
        else:
            raise ValueError(
                "Tried to create histogram/n-tuple, but received" "invalid type"
            )

    def is_expert(self):
        """!
        @return Returns true if this plotuple has the expert option
        """
        return not self._mop.has_option("shifter")

    def perform_comparison(self):
        """!
        Takes the reference (self.reference.object) and the newest revision
        (self.newest.object) and a canvas. Performs a comparison of the
        two objects.
        @return: None
        """

        tester = validationcomparison.get_comparison(
            self._reference.object, self._newest.object, self._mop
        )

        self._comparison_result_long = tester.comparison_result_long.format(
            revision1=self._reference.revision, revision2=self._newest.revision
        )
        self.comparison_result = tester.comparison_result

    def _set_background(self, canvas):

        # kRed         #FF0000 Red
        # kRed - 9     #FF9999 Sweet pink

        # kOrange + 1  #FF9832 Sun
        # kOrange - 9  #FFCC9A Manhattan

        # kGreen - 3   #33CC33 Lime green
        # kGreen - 10  #CCFFCC Chinook

        # kAzure       #0032FE Medium blue
        # kAzure - 2   #3265FE Medium slate blue
        # kAzure - 9   #98CBFF Jordy blue

        colors = {
            "error": ROOT.kRed,
            "warning": ROOT.kOrange + 1,
            "equal": ROOT.kGreen - 3,
            "not_compared": ROOT.kAzure - 2,
        }
        colors_expert = {
            "error": ROOT.kRed - 9,
            "warning": ROOT.kOrange - 9,
            "equal": ROOT.kGreen - 10,
            "not_compared": ROOT.kAzure - 9,
        }

        if self.is_expert():
            color = colors_expert[self.comparison_result]
        else:
            color = colors[self.comparison_result]

        canvas.SetFillColor(color)
        canvas.GetFrame().SetFillColor(ROOT.kWhite)

    def _draw_ref(self, canvas):
        """!
        Takes the reference RootObject (self.reference.object)
        and a (sub)canvas and plots it with the correct line-style etc.
        @param canvas: Reference to the canvas on which we will draw the
            reference object.
        @return. None
        """
        self._remove_stats_tf1(self._reference.object)

        # Line is thick and black
        self._reference.object.SetLineColor(ROOT.kBlack)
        self._reference.object.SetLineWidth(2)
        self._reference.object.SetLineStyle(1)

        # Area under the curve is solid gray
        self._reference.object.SetFillColor(ROOT.kGray)
        self._reference.object.SetFillStyle(1001)

        # Draw the reference on the canvas
        self._draw_root_object(
            self.type,
            self._reference.object,
            self._reference.object.GetOption(),
        )
        canvas.Update()
        canvas.GetFrame().SetFillColor(ROOT.kWhite)

    @staticmethod
    def _remove_stats_tf1(obj):
        # removed TF1s which might have been added by validation scripts
        # in tracking/scripts/tracking/validation/plot.py:1597
        tf1 = obj.FindObject("FitAndStats")
        if tf1:
            function_list = obj.GetListOfFunctions()
            function_list.Remove(tf1)

    # TODO: is this actually used or can it be removed ?
    def create_image_plot(self):
        """!
        Creates image plot for TASImage-objects.
        @return: None
        """

        # Create a ROOT canvas on which we will draw our histograms
        self._width = 700
        if len(self._elements) > 4:
            canvas = ROOT.TCanvas("", "", 700, 1050)
            self._height = 1050
        else:
            canvas = ROOT.TCanvas("", "", 700, 525)
            self._height = 525

        # Split the canvas into enough parts to fit all image_objects
        # Find numbers x and y so that x*y = N (number of histograms to be
        # plotted), and x,y close to sqrt(N)

        if len(self._root_objects) == 1:
            x = y = 1
        elif len(self._root_objects) == 2:
            x = 2
            y = 1
        else:
            x = 2
            y = int(math.floor((len(self._root_objects) + 1) / 2))

        # Actually split the canvas and go to the first pad ('sub-canvas')
        canvas.Divide(x, y)
        pad = canvas.cd(1)
        pad.SetFillColor(ROOT.kWhite)

        # If there is a reference object, plot it first
        if self._reference is not None:
            self._draw_ref(pad)

        # Now draw the normal plots
        items_to_plot_count = len(self._elements)
        for plot in reversed(self._elements):

            # Get the index of the current plot
            index = index_from_revision(plot.revision, self._work_folder)
            style = get_style(index, items_to_plot_count)

            self._remove_stats_tf1(plot.object)

            # Set line properties accordingly
            plot.object.SetLineColor(style.GetLineColor())
            plot.object.SetLineWidth(style.GetLineWidth())
            plot.object.SetLineStyle(style.GetLineStyle())

            # Switch to the correct sub-panel of the canvas. If a ref-plot
            # exists, we have to go one panel further compared to the
            # no-ref-case
            if self._reference is not None:
                i = 2
            else:
                i = 1

            pad = canvas.cd(self._elements.index(plot) + i)
            pad.SetFillColor(ROOT.kWhite)

            # Draw the reference on the canvas
            self._draw_root_object(
                self.type, plot.object, plot.object.GetOption()
            )
            pad.Update()
            pad.GetFrame().SetFillColor(ROOT.kWhite)

            # Write the title in the correct color
            title = pad.GetListOfPrimitives().FindObject("title")
            if title:
                title.SetTextColor(style.GetLineColor())

        # Save the plot as PNG and PDF
        canvas.Print(os.path.join(self._plot_folder, self.get_png_filename()))
        canvas.Print(os.path.join(self._plot_folder, self.get_pdf_filename()))

        self._file = os.path.join(
            self._plot_folder,
            "{}_{}".format(strip_ext(self.rootfile), self.key),
        )

    def get_png_filename(self):
        return "{}_{}.png".format(strip_ext(self.rootfile), self.key)

    def get_pdf_filename(self):
        return "{}_{}.pdf".format(strip_ext(self.rootfile), self.key)

    @staticmethod
    def _draw_root_object(typ, obj, options):
        """
        Special handling of the ROOT Draw calls, as some
        ROOT objects have a slightly differen flavour.
        """

        if typ == "TEfficiency" or typ == "TGraph":
            # TEff does not provide DrawCopy
            obj.Draw(options)
        else:
            obj.DrawCopy(options)

    def create_histogram_plot(self, mode):
        """!
        Plots all histogram-objects in this Plotuple together in one histogram,
        which is then given the name of the key.
        @param mode: Determines whether it is a one- or
        two-dimensional histogram.
            Accepted values are '1D' and '2D'
        @return: None
        """

        # If we don't get a valid 'mode', we can stop right here
        if mode not in ["1D", "2D"]:
            return

        # Create a ROOT canvas on which we will draw our histograms
        self._width = 700
        if mode == "2D" and len(self._elements) > 4:
            self._height = 1050
        else:
            self._height = 525
        canvas = ROOT.TCanvas("", "", self._width, self._height)

        # Create a ROOT Legend
        legend = ROOT.TLegend(0.01, 0.01, 0.53, 0.06)
        legend.SetNColumns(3)
        legend.SetTextSize(0.02)

        # Allow possibility to turn off the stats box
        if self._mop.has_option("nostats"):
            ROOT.gStyle.SetOptStat("")
        else:
            ROOT.gStyle.SetOptStat("nemr")

        # If there is a reference object, and the list of plots is not empty,
        # perform a Chi^2-Test on the reference object and the first object in
        # the plot list:
        if (
            self._reference is not None
            and self._newest
            and not self._reference == self._newest
        ):
            self.perform_comparison()

        # A variable which holds whether we
        # have drawn on the canvas already or not
        # (only used for the 1D case)
        drawn = False

        # Now we distinguish between 1D and 2D histograms
        # If we have a 1D histogram
        if mode == "1D":

            if not self._mop.has_option("nogrid"):
                canvas.SetGrid()
            if self._mop.has_option("logx"):
                canvas.SetLogx()
            if self._mop.has_option("logy"):
                canvas.SetLogy()

            # If there is a reference object, plot it first
            if self._reference is not None:
                self._draw_ref(canvas)
                drawn = True

        # If we have a 2D histogram
        elif mode == "2D":

            # Split the canvas into enough parts to fit all histogram_objects
            # Find numbers x and y so that x*y = N (number of histograms to be
            # plotted), and x,y close to sqrt(N)

            if len(self._root_objects) == 1:
                x = y = 1
            elif len(self._root_objects) == 2:
                x = 2
                y = 1
            else:
                x = 2
                y = int(math.floor((len(self._root_objects) + 1) / 2))

            # Actually split the canvas and go to the first pad ('sub-canvas')
            canvas.Divide(x, y)
            pad = canvas.cd(1)
            pad.SetFillColor(ROOT.kWhite)

            # If there is a reference object, plot it first
            if self._reference is not None:
                self._draw_ref(pad)

        items_to_plot_count = len(self._elements)

        # Now draw the normal plots
        for plot in reversed(self._elements):

            # Get the index of the current plot
            index = index_from_revision(plot.revision, self._work_folder)
            style = get_style(index, items_to_plot_count)

            self._remove_stats_tf1(plot.object)

            # Set line properties accordingly
            plot.object.SetLineColor(style.GetLineColor())
            plot.object.SetLineWidth(style.GetLineWidth())
            plot.object.SetLineStyle(style.GetLineStyle())

            # If we have a one-dimensional histogram
            if mode == "1D":
                if not drawn:
                    # Get additional options for 1D histograms
                    # (Intersection with self.metaoptions)
                    additional_options = ["C"]
                    additional_options = [
                        option
                        for option in additional_options
                        if self._mop.has_option(option)
                    ]
                    options_str = plot.object.GetOption() + " ".join(
                        additional_options
                    )
                    drawn = True
                else:
                    options_str = "SAME"

                legend.AddEntry(plot.object, plot.revision)

                self._draw_root_object(self.type, plot.object, options_str)

                # redraw grid ontop of histogram, if selected
                if not self._mop.has_option("nogrid"):
                    canvas.RedrawAxis("g")

                canvas.Update()
                canvas.GetFrame().SetFillColor(ROOT.kWhite)

            # If we have a two-dimensional histogram
            elif mode == "2D":
                # Switch to the correct sub-panel of the canvas. If a ref-plot
                # exists, we have to go one panel further compared to the
                # no-ref-case
                if self._reference is not None:
                    i = 2
                else:
                    i = 1

                pad = canvas.cd(self._elements.index(plot) + i)
                pad.SetFillColor(ROOT.kWhite)

                # Get additional options for 2D histograms
                additional_options = ""
                for _ in ["col", "colz", "cont", "contz", "box"]:
                    if self._mop.has_option(_):
                        additional_options += " " + _

                # Draw the reference on the canvas
                self._draw_root_object(
                    self.type,
                    plot.object,
                    plot.object.GetOption() + additional_options,
                )
                pad.Update()
                pad.GetFrame().SetFillColor(ROOT.kWhite)

                # Write the title in the correct color
                title = pad.GetListOfPrimitives().FindObject("title")
                if title:
                    title.SetTextColor(style.GetLineColor())

        if self._newest:
            # if there is at least one revision
            self._set_background(canvas)

        canvas.GetFrame().SetFillColor(ROOT.kWhite)

        # Add reference legend entry last for neatness
        if self._reference is not None:
            legend.AddEntry(self._reference.object, 'reference')

        legend.Draw()

        # Save the plot as PNG and PDF
        canvas.Print(os.path.join(self._plot_folder, self.get_png_filename()))
        canvas.Print(os.path.join(self._plot_folder, self.get_pdf_filename()))

        self._file = os.path.join(
            self._plot_folder,
            "{}_{}".format(strip_ext(self.rootfile), self.key),
        )

    def create_graph_plot(self):
        """!
        Plots as TGraph/TGraphErrors
        @return: None
        """

        # Create a ROOT canvas on which we will draw our plots
        self._width = 700
        self._height = 525
        canvas = ROOT.TCanvas("", "", self._width, self._height)

        # Allow possibility to turn off the stats box
        if self._mop.has_option("nostats"):
            ROOT.gStyle.SetOptStat("")
        else:
            ROOT.gStyle.SetOptStat("nemr")

        # If there is a reference object, and the list of plots is not empty,
        # perform a Chi^2-Test on the reference object and the first object in
        # the plot list:
        if (
            self._reference is not None
            and self._newest
            and not self._reference == self._newest
        ):
            self.perform_comparison()

        if not self._mop.has_option("nogrid"):
            canvas.SetGrid()
        if self._mop.has_option("logx"):
            canvas.SetLogx()
        if self._mop.has_option("logy"):
            canvas.SetLogy()

        # A variable which holds whether we
        # have drawn on the canvas already or not
        drawn = False

        # If there is a reference object, plot it first
        if self._reference is not None:
            self._draw_ref(canvas)
            drawn = True

        items_to_plot_count = len(self._elements)
        # Now draw the normal plots
        for plot in reversed(self._elements):

            # Get the index of the current plot
            index = index_from_revision(plot.revision, self._work_folder)
            style = get_style(index, items_to_plot_count)

            # self.remove_stats_tf1(plot.object)

            # Set line properties accordingly
            plot.object.SetLineColor(style.GetLineColor())
            plot.object.SetLineWidth(style.GetLineWidth())
            plot.object.SetLineStyle(style.GetLineStyle())

            # If we have a one-dimensional histogram
            if not drawn:

                # todo: refactor like in plot hist
                # Get additional options for 1D histograms
                additional_options = ""
                for _ in ["C"]:
                    if self._mop.has_option(_):
                        additional_options += " " + _

                # Draw the reference on the canvas
                self._draw_root_object(
                    self.type,
                    plot.object,
                    plot.object.GetOption() + additional_options,
                )
                drawn = True
            else:
                self._draw_root_object(self.type, plot.object, "SAME")

            # redraw grid ontop of histogram, if selected
            if not self._mop.has_option("nogrid"):
                canvas.RedrawAxis("g")

            canvas.Update()
            canvas.GetFrame().SetFillColor(ROOT.kWhite)

        if self._newest:
            # if there is at least one revision
            self._set_background(canvas)

        # Save the plot as PNG and PDF
        canvas.Print(os.path.join(self._plot_folder, self.get_png_filename()))
        canvas.Print(os.path.join(self._plot_folder, self.get_pdf_filename()))

        self._file = os.path.join(
            self._plot_folder,
            "{}_{}".format(strip_ext(self.rootfile), self.key),
        )

    def create_html_content(self):

        # self.elements
        self._html_content = ""

        for elem in self._elements:
            self._html_content += (
                "<p>" + elem.revision + "</p>" + elem.object.GetTitle()
            )

        # there is no file storing this, because it is directly in the json
        # file
        self._file = None

    def get_meta_information(self):
        assert self.type == "meta"
        key = self._newest.object.GetName().strip().lower()
        value = self._newest.object.GetTitle()
        return key, value

    def create_ntuple_table_json(self):
        """!
        If the Plotuple-object contains n-tuples, this will create the
        a JSON file, which is later converted to HTML by the javascript
        function fillNtupleTable.
        """

        json_nutple = {}

        # The dictionary will have the following form
        # {
        #     "reference (if exist)": [
        #         ('variable 1', 'reference value for variable 1'),
        #         ('variable 2', 'reference value for variable 2'),
        #         ...
        #     ],
        #     "revision": [
        #         ...
        #     ]
        # }

        precision = self._mop.int_value("float-precision", default=4)
        format_str = f"{{0:.{precision}f}}"

        def value2str(obj):
            # assuming that I have a float
            return format_str.format(obj)

        colum_names = []
        for key in list(self._newest.object.keys()):
            colum_names.append(key)

        # If there is a reference object, print the reference values as the
        # first row of the table
        if self._reference and "reference" in self._revisions:
            json_nutple["reference"] = []

            key_list = list(self._reference.object.keys())
            for column in colum_names:
                if column in key_list:
                    value_str = value2str(self._reference.object[column])
                    json_nutple["reference"].append((column, value_str))
                else:
                    json_nutple["reference"].append((column, None))

        # Now print the values for all other revisions
        for ntuple in self._elements:
            if ntuple.revision not in json_nutple:
                json_nutple[ntuple.revision] = []

            for column in colum_names:
                if column in ntuple.object:
                    value_str = value2str(ntuple.object[column])
                    json_nutple[ntuple.revision].append((column, value_str))
                else:
                    json_nutple[ntuple.revision].append((column, None))

        json_ntuple_file = os.path.join(
            self._plot_folder,
            "{}_{}.json".format(strip_ext(self.rootfile), self.key),
        )

        with open(json_ntuple_file, "w+") as json_file:
            json.dump(json_nutple, json_file)

        self._file = json_ntuple_file

    def get_plot_title(self):
        if self._file:
            return os.path.basename(self._file).replace(".", "_").strip()
        else:
            # this is for html content which is not stored in any file
            return self.key

    def create_json_object(self):
        if self.type == "TNtuple":
            return json_objects.ComparisonNTuple(
                title=self.get_plot_title(),
                description=self._description,
                contact=self._contact,
                check=self._check,
                is_expert=self.is_expert(),
                json_file_path=os.path.relpath(
                    self._file,
                    validationpath.get_html_folder(self._work_folder),
                ),
            )
        elif self.type == "TNamed":
            return json_objects.ComparisonHtmlContent(
                title=self.get_plot_title(),
                description=self._description,
                contact=self._contact,
                check=self._check,
                is_expert=self.is_expert(),
                html_content=self._html_content,
            )
        elif self.type == "meta":
            return None
        else:
            return json_objects.ComparisonPlot(
                title=self.get_plot_title(),
                comparison_result=self.comparison_result,
                comparison_text=self._comparison_result_long,
                description=self._description,
                contact=self._contact,
                check=self._check,
                height=self._height,
                width=self._width,
                is_expert=self.is_expert(),
                plot_path=os.path.relpath(
                    self._plot_folder,
                    validationpath.get_html_folder(self._work_folder),
                )
                + "/",
                png_filename=self.get_png_filename(),
                pdf_filename=self.get_pdf_filename(),
                warnings=self.warnings,
            )


"""
Parser function parse() to parse the <seedname>-shc-{fermiscan,freqscan}.dat output file of postw90.x.

"""
import inspect
import re
from collections import defaultdict

from . import show_output

def parse(fname):
    """
    Open the file, parses it and return the values.
    """
    retdict = defaultdict(list)

    if show_output:
        print("[{}.{}] Parsing file '{}'".format(
            __name__, inspect.currentframe().f_code.co_name, fname))

    with open(fname) as f:
        lines = f.readlines()

    for lno, l in enumerate(lines):

        pieces = l.split()

        if lno == 0:
            if len(pieces) < 2:
                raise ValueError("Wrong SHC dat file: header is wrong")
            else:
                if pieces[1] == 'Fermi':
                    freqscan = False
                    continue
                elif pieces[1] == 'Frequency(eV)':
                    freqscan = True
                    continue
                else:
                    raise ValueError("Wrong SHC dat file: header is wrong")

        if len(pieces) == 0:
            # skip blank line
            continue

        if freqscan:
            if len(pieces) == 4 :
                retdict['frequency'].append(float(pieces[1]))
                retdict['shc_re'].append(float(pieces[2]))
                retdict['shc_im'].append(float(pieces[3]))
            else:
                raise ValueError("Wrong line length ({}, instead of 4); line content: {}".format(
                    len(pieces), l))
        else:
            if len(pieces) == 3 :
                retdict['energy'].append(float(pieces[1]))
                retdict['shc'].append(float(pieces[2]))
            else:
                raise ValueError("Wrong line length ({}, instead of 3); line content: {}".format(
                    len(pieces), l))

    retdict = dict(retdict)
    if show_output:
        for k in sorted(retdict):
            print("  {}: {}".format(k, retdict[k]))
        print("-"*72)
    return retdict


from queue import deque
from collections import defaultdict

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def deepestLeavesSum(self, root: Optional[TreeNode]) -> int:

        stack = deque()
        level = 0
        stack.append((level, root))
        d = defaultdict(list)
        while stack:
            level, node = stack.popleft()
            d[level].append(node)
            level += 1
            for child in [node.left, node.right]:
                if child is not None:
                    stack.append((level, child))
        return sum([node.val for node in d[level-1]])


import pytest
import shutil
import numpy as np
from concrete.compiler import (
    LibrarySupport,
    PublicArguments,
    SimulatedValueExporter,
    SimulatedValueDecrypter,
    CompilationOptions,
)

def assert_result(result, expected_result):
    """Assert that result and expected result are equal.

    result and expected_result can be integers on numpy arrays.
    """
    assert type(expected_result) == type(result)
    if isinstance(expected_result, int):
        assert result == expected_result
    else:
        assert np.all(result == expected_result)

def run_simulated(engine, args_and_shape, compilation_result):
    client_parameters = engine.load_client_parameters(compilation_result)
    sim_value_exporter = SimulatedValueExporter.new(client_parameters)
    values = []
    pos = 0
    for arg, shape in args_and_shape:
        if shape is None:
            assert isinstance(arg, int)
            values.append(sim_value_exporter.export_scalar(pos, arg))
        else:
            assert isinstance(arg, list)
            assert isinstance(shape, list)
            values.append(sim_value_exporter.export_tensor(pos, arg, shape))
        pos += 1
    public_arguments = PublicArguments.new(client_parameters, values)
    server_lambda = engine.load_server_lambda(compilation_result)
    public_result = engine.simulate(server_lambda, public_arguments)
    sim_value_decrypter = SimulatedValueDecrypter.new(client_parameters)
    result = sim_value_decrypter.decrypt(0, public_result.get_value(0))
    return result

def compile_run_assert(
    engine,
    mlir_input,
    args_and_shape,
    expected_result,
    options=CompilationOptions.new("main"),
):
    # compile with simulation
    options.simulation(True)
    compilation_result = engine.compile(mlir_input, options)
    result = run_simulated(engine, args_and_shape, compilation_result)
    assert_result(result, expected_result)

end_to_end_fixture = [
    pytest.param(
        """
            func.func @main(%arg0: !FHE.eint<7>, %arg1: i8) -> !FHE.eint<7> {
                %1 = "FHE.add_eint_int"(%arg0, %arg1): (!FHE.eint<7>, i8) -> (!FHE.eint<7>)
                return %1: !FHE.eint<7>
            }
            """,
        (5, 7),
        12,
        id="add_eint_int",
    ),
    pytest.param(
        """
            func.func @main(%arg0: !FHE.eint<7>) -> !FHE.eint<7> {
                %tlu = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127]> : tensor<128xi64>
                %1 = "FHE.apply_lookup_table"(%arg0, %tlu): (!FHE.eint<7>, tensor<128xi64>) -> (!FHE.eint<7>)
                return %1: !FHE.eint<7>
            }
            """,
        (73,),
        73,
        id="apply_lookup_table",
    ),
    pytest.param(
        """
            func.func @main(%arg0: tensor<4x!FHE.eint<7>>, %arg1: tensor<4xi8>) -> !FHE.eint<7>
            {
                %ret = "FHELinalg.dot_eint_int"(%arg0, %arg1) :
                    (tensor<4x!FHE.eint<7>>, tensor<4xi8>) -> !FHE.eint<7>
                return %ret : !FHE.eint<7>
            }
            """,
        (
            np.array([1, 2, 3, 4], dtype=np.uint8),
            np.array([4, 3, 2, 1], dtype=np.uint8),
        ),
        20,
        id="dot_eint_int_uint8",
    ),
    pytest.param(
        """
            func.func @main(%a0: tensor<4x!FHE.eint<6>>, %a1: tensor<4xi7>) -> tensor<4x!FHE.eint<6>> {
                %res = "FHELinalg.add_eint_int"(%a0, %a1) : (tensor<4x!FHE.eint<6>>, tensor<4xi7>) -> tensor<4x!FHE.eint<6>>
                return %res : tensor<4x!FHE.eint<6>>
            }
            """,
        (
            np.array([31, 6, 12, 9], dtype=np.uint8),
            np.array([32, 9, 2, 3], dtype=np.uint8),
        ),
        np.array([63, 15, 14, 12]),
        id="add_eint_int_1D",
    ),
    pytest.param(
        """
            func.func @main(%arg0: !FHE.esint<7>) -> !FHE.esint<7> {
                %0 = "FHE.neg_eint"(%arg0): (!FHE.esint<7>) -> !FHE.esint<7>
                return %0: !FHE.esint<7>
            }
            """,
        (5,),
        -5,
        id="neg_eint_signed",
    ),
    pytest.param(
        """
            func.func @main(%arg0: tensor<2x!FHE.esint<7>>) -> tensor<2x!FHE.esint<7>> {
                %0 = "FHELinalg.neg_eint"(%arg0): (tensor<2x!FHE.esint<7>>) -> tensor<2x!FHE.esint<7>>
                return %0: tensor<2x!FHE.esint<7>>
            }
            """,
        (np.array([-5, 3]),),
        np.array([5, -3]),
        id="neg_eint_signed_2",
    ),
    pytest.param(
        """
        func.func @main(%arg0: tensor<4x4x!FHE.eint<13>>, %arg1: tensor<4x4xi14>) -> tensor<4x4x!FHE.eint<13>> {
            %0 = "FHELinalg.matmul_eint_int"(%arg0, %arg1) : (tensor<4x4x!FHE.eint<13>>, tensor<4x4xi14>) -> tensor<4x4x!FHE.eint<13>>
            %1 = "FHELinalg.matmul_eint_int"(%0, %arg1) : (tensor<4x4x!FHE.eint<13>>, tensor<4x4xi14>) -> tensor<4x4x!FHE.eint<13>>
            %2 = "FHELinalg.matmul_eint_int"(%1, %arg1) : (tensor<4x4x!FHE.eint<13>>, tensor<4x4xi14>) -> tensor<4x4x!FHE.eint<13>>
            %3 = "FHELinalg.matmul_eint_int"(%2, %arg1) : (tensor<4x4x!FHE.eint<13>>, tensor<4x4xi14>) -> tensor<4x4x!FHE.eint<13>>
            return %3 : tensor<4x4x!FHE.eint<13>>
        }
        """,
        (
            np.array([i // 4 for i in range(16)]).reshape((4, 4)),
            np.array([i // 4 for i in range(15, -1, -1)]).reshape((4, 4)),
        ),
        np.array(
            [
                0,
                0,
                0,
                0,
                1296,
                1296,
                1296,
                1296,
                2592,
                2592,
                2592,
                2592,
                3888,
                3888,
                3888,
                3888,
            ]
        ).reshape((4, 4)),
        id="matul_chain_with_crt",
    ),
    pytest.param(
        """
            func.func @main(%arg0: !FHE.eint<14>, %arg1: tensor<16384xi64>) -> !FHE.eint<14> {
                %cst = arith.constant 15 : i15
                %v = "FHE.add_eint_int"(%arg0, %cst): (!FHE.eint<14>, i15) -> (!FHE.eint<14>)
                %1 = "FHE.apply_lookup_table"(%v, %arg1): (!FHE.eint<14>, tensor<16384xi64>) -> (!FHE.eint<14>)
                return %1: !FHE.eint<14>
            }
        """,
        (
            81,
            np.array(range(16384), dtype=np.uint64),
        ),
        96,
        id="add_lut_crt",
    ),
]

end_to_end_parallel_fixture = [
    pytest.param(
        """
            func.func @main(%x: tensor<3x4x!FHE.eint<7>>, %y: tensor<3x4x!FHE.eint<7>>) -> tensor<3x2x!FHE.eint<7>> {
                %c = arith.constant dense<[[1, 2], [3, 4], [5, 0], [1, 2]]> : tensor<4x2xi8>
                %0 = "FHELinalg.matmul_eint_int"(%x, %c): (tensor<3x4x!FHE.eint<7>>, tensor<4x2xi8>) -> tensor<3x2x!FHE.eint<7>>
                %1 = "FHELinalg.matmul_eint_int"(%y, %c): (tensor<3x4x!FHE.eint<7>>, tensor<4x2xi8>) -> tensor<3x2x!FHE.eint<7>>
                %2 = "FHELinalg.add_eint"(%0, %1): (tensor<3x2x!FHE.eint<7>>, tensor<3x2x!FHE.eint<7>>) -> tensor<3x2x!FHE.eint<7>>
                return %2 : tensor<3x2x!FHE.eint<7>>
            }
            """,
        (
            np.array([[1, 2, 3, 4], [4, 2, 1, 0], [2, 3, 1, 5]], dtype=np.uint8),
            np.array([[1, 2, 3, 4], [4, 2, 1, 1], [2, 3, 1, 5]], dtype=np.uint8),
        ),
        np.array([[52, 36], [31, 34], [42, 52]]),
        id="matmul_eint_int_uint8",
    ),
    pytest.param(
        """
            func.func @main(%a0: tensor<4x!FHE.eint<6>>, %a1: tensor<4xi7>, %a2: tensor<4x!FHE.eint<6>>, %a3: tensor<4xi7>) -> tensor<4x!FHE.eint<6>> {
                %1 = "FHELinalg.add_eint_int"(%a0, %a1) : (tensor<4x!FHE.eint<6>>, tensor<4xi7>) -> tensor<4x!FHE.eint<6>>
                %2 = "FHELinalg.add_eint_int"(%a2, %a3) : (tensor<4x!FHE.eint<6>>, tensor<4xi7>) -> tensor<4x!FHE.eint<6>>
                %res = "FHELinalg.add_eint"(%1, %2) : (tensor<4x!FHE.eint<6>>, tensor<4x!FHE.eint<6>>) -> tensor<4x!FHE.eint<6>>
                return %res : tensor<4x!FHE.eint<6>>
            }
            """,
        (
            np.array([1, 2, 3, 4], dtype=np.uint8),
            np.array([9, 8, 6, 5], dtype=np.uint8),
            np.array([3, 2, 7, 0], dtype=np.uint8),
            np.array([1, 4, 2, 11], dtype=np.uint8),
        ),
        np.array([14, 16, 18, 20]),
        id="add_eint_int_1D",
    ),
]

@pytest.mark.parametrize("mlir_input, args, expected_result", end_to_end_fixture)
def test_lib_compile_and_run_simulation(mlir_input, args, expected_result):
    artifact_dir = "./py_test_lib_compile_and_run"
    engine = LibrarySupport.new(artifact_dir)
    args_and_shape = []
    for arg in args:
        if isinstance(arg, int):
            args_and_shape.append((arg, None))
        else:  # np.array
            args_and_shape.append((arg.flatten().tolist(), list(arg.shape)))
    compile_run_assert(engine, mlir_input, args_and_shape, expected_result)
    shutil.rmtree(artifact_dir)


# -*- coding: utf-8 -*-
"""
NAME
    download_gfs
KEYWORD ARGUMENTS
    Start and End date (Format: YYYYMMDD)
    Earliest date 1st January 2007 (20070101),
    there are no weather data before that date to download (status May 2019)
KEYWORD ARGUMENTS optional
    path in which the file should be placed
DESCRIPTION
    download_gfs downloads weather data (analysis 0.5 degree grid) from nomads.ncdc.noaa.gov
    linux os is required
NOTE
    The files will not be renamed and not be checked if the download succeded!

    Comments / improvements are welcome.

CONTACT
    florian.geyer@zamg.ac.at
    Version 1, May 2019

DISCLAIMER
    This software has been developed for flexpart workshop 2019 @ ZAMG

"""

def check_date(date):
    """
        checks if date is really a date
    """
    import datetime
    correctDate = None
    date = str(date)

    if (len(date)!=8):
        return False
    year = int(date[0:4])
    month = int(date[4:6])
    day = int(date[6:8])
    try:
        datetime.datetime(year,month,day)
        correctDate = True
    except ValueError:
        correctDate = False
    return correctDate

def convert_str2date(date):
    """
        converts date (as integer) to python datetime
    """
    import datetime
    date = str(date)
    year = int(date[0:4])
    month = int(date[4:6])
    day = int(date[6:8])
    return datetime.datetime(year,month,day)

def create_filenames(date):
    """
        creates filenames for givn date,
        from the runs at 0, 6, 12 and 18 UTC
        only t=0 and t=3 will be downloaded
    """

    filelist = list()
    for t in ["0000", "0600", "1200", "1800"]:
        for t2 in ["000", "003"]:
            filelist.append("gfsanl_4_"+d.strftime('%Y%m%d')+"_"+t+"_"+t2+".grb2")
    return filelist

def download_file(path, filename, destination):
    """
        downloads files from path and saves to destination
    """
    import os
    command = "wget -q -O "+destination+"/"+filename+" ftp://nomads.ncdc.noaa.gov/"+path+"/"+filename
    os.system(command)

def check_path(path):
    """
        checks if path exists
    """
    import os
    if not os.path.exists(path):
        print("Path does not exist")
        print("")
        sys.exit()

def start_ftp_date():
    """
        from this date 0.5 degree grid data exists on ftp
    """
    return 20070101

import sys
from ftplib import FTP
import datetime

print("")

# checking input arguments (1 and 2 are start and end date
try:
    start = sys.argv[1]
    end   = sys.argv[2]
except:
    print("Please give me a start and end date (format: YYYYMMDD)")
    print("")
    sys.exit()

# checking destination (optinal), if no destination givn,
# files will be saved in current directory
try:
    destination = sys.argv[3]
    if (destination[-1]!="/"):
      destination = destination+"/"
except:
    destination = "./"

# print input arguments
print("Start: ", start)
print("End:   ", end)
print("Path: ", destination)
print("")

check_path(destination)

start_ftp_date = start_ftp_date()

# check if dates are in correct format
if (check_date(start)==False):
    print("Wrong start date, please try again (Format should be YYYYMMDD)!")
    print("")
    sys.exit()
if (check_date(end)==False):
    print("Wrong end date, please try again (Format should be YYYYMMDD)!")
    print("")
    sys.exit()
start_date = convert_str2date(start)
end_date   = convert_str2date(end)
start_ftp_date   = convert_str2date(start_ftp_date)

# check if dates are in correct order
if (start_date>end_date):
    print("Your start date was after the end date, I will turn it around!")
    print("")
    start_date, end_date = end_date, start_date

if (start_date<start_ftp_date):
    print("The FTP-Server only provides data after the 1st January 2007, please choose another date!")
    print("")
    sys.exit()

print("start downloading from ", start_date, "to", end_date)

# create date list
date_list = [start_date + datetime.timedelta(days=x) for x in range(0, (end_date-start_date).days+1)]

# connect to ftp
ftp = FTP('nomads.ncdc.noaa.gov')
ftp.login()

# iter through date_list and download
for d in date_list:
    path = "GFS/analysis_only/"+d.strftime('%Y%m')+"/"+d.strftime('%Y%m%d')+"/"
    try:
        ftp.cwd(path)
        print(path, "found")
        file_list_ftp = ftp.nlst()
        file_list = create_filenames(d)
        for f in file_list:
            if f in file_list_ftp:
                print("  ", f, "found, download as "+destination+f)
                download_file(path, f, destination)
            else:
                print("  ", f, "not found")
        ftp.cwd("../../../../")
    except:
        print(d.strftime('%Y/%m/%d'), "Directory not found")
ftp.quit()
print("")


from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import DetailView, TemplateView

from drchrono.endpoints import AllergyEndpoint, MedicationEndpoint
from drchrono.models import Doctor, Patient

class PatientListView(LoginRequiredMixin, TemplateView):

    template_name = "patient_list.html"

    def get_context_data(self, **kwargs):
        kwargs = super(PatientListView, self).get_context_data(**kwargs)
        doctor = Doctor.objects.first()
        patients = Patient.objects.filter(doctor_id=doctor.api_id).order_by("last_name")
        kwargs["doctor"] = doctor
        kwargs["patients"] = patients
        return kwargs

class PatientDetailView(LoginRequiredMixin, DetailView):

    model = Patient
    template_name = "patient.html"

    def get_context_data(self, **kwargs):
        kwargs = super(PatientDetailView, self).get_context_data(**kwargs)
        allergies_api = AllergyEndpoint(self.request.session["access_token"])
        allergies = allergies_api.list(params={"patient": kwargs["patient"].api_id})
        medications_api = MedicationEndpoint(self.request.session["access_token"])
        medications = medications_api.list(
            params={
                "patient": kwargs["patient"].api_id,
                "status": "active"
            }
        )
        kwargs["allergies"] = allergies
        kwargs["medications"] = medications
        return kwargs


from django.contrib import admin

# Register your models here.
from .models import EmployeeDetail

class EmployeeAdmin(admin.ModelAdmin):
    list_display = ["user_id" , "created" , "updated"]

    class Meta:
        model = EmployeeDetail

admin.site.register(EmployeeDetail,EmployeeAdmin)


#!/usr/bin/env python

import importlib
import os
import sys
import datetime

from lib.attachment_events_reader import AttachmentEventsReader
from lib.config import Config
from lib.imap import Mailbox
# from lib.processed import Processed
from lib.processed import Processed
from lib.receipts import receipts

# 0. error handling: add date+time to output
def exception_add_datetime(exctype, value, traceback):
    sys.stderr.write("[ERROR][" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "] \n")
    sys.__excepthook__(exctype, value, traceback)
sys.excepthook = exception_add_datetime

app_path = os.path.dirname(os.path.realpath(__file__))

# 1. connect to mailbox
config = Config(app_path + '/.env')
mailbox = Mailbox(config)

# 2. fetch emails with a csv attachment
messages = mailbox.messages(attachment_suffix_filter='csv')
if messages is None:
    sys.exit(0)

# 3. turn attachments into timelines events
db_dir = app_path + '/storage/processed'
if config.get('timelines_events_duplicate_detection'):
    processed = Processed('timelines_events', db_dir)
    reader = AttachmentEventsReader(processed)
else:
    processed = None

# 4. process messages
backend_module = importlib.import_module(config.get('backend_module'))
backend_class = getattr(backend_module, config.get('backend_class'))
backend = backend_class()
for message in messages:
    # try:
    reader = AttachmentEventsReader()
    # read all relevant attachments
    for attachment in message['attachments']:
        reader.add(message, attachment_payload=attachment['payload'])
    # deliver events to backend
    delivery_status_lines_list = []
    for event in reader.events:
        project_id = event.project_id()
        if project_id is not None and project_id != '':
            event.delivery_status_lines = backend.deliver_timelines_event(event)
        if processed is not None:
            processed.now(event)
    # send configured receipts
    if config.get('receipt_to_sender') or config.get('receipt_copy_to_addresses') != '':
        receipts(config).send(message, reader.events)

# 5. move successfully processed emails to processed folder
if config.get('imap_move_processed_messages'):
    mailbox.move_to_processed_folder(messages)

# 6. disconnect from mailbox
mailbox.close()


'''
References:
    https://www.ritchieng.com/machine-learning-evaluate-classification-model/
    https://datatofish.com/logistic-regression-python/
    https://stackabuse.com/gradient-descent-in-python-implementation-and-theory/
'''

from sklearn.linear_model import LogisticRegression
import pandas as pd

#import data
x_train = pd.read_csv(r'C:/Users/Berto/data for homework 3/Problem 6_data_trainx.csv')
y_train = pd.read_csv(r'C:/Users/Berto/data for homework 3/Problem 6_data_trainy.csv')

x_test = pd.read_csv(r'C:/Users/Berto/data for homework 3/Problem 6_data_testx.csv')
y_test = pd.read_csv(r'C:/Users/Berto/data for homework 3/Problem 6_data_testy.csv')

#combine data
frames_train = [x_train, y_train]
df_train = pd.concat(frames_train, axis=1, join='inner')

frames_test = [x_test, y_test]
df_test = pd.concat(frames_test, axis=1, join='inner')

#separate data
X_train = df_train[['x1', 'x2']]
y_train = df_train['y1']

X_test = df_test[['x1', 'x2']]
y_test = df_test['y1']

#convert dataframe to array
X_train, y_train = X_train.to_numpy(), y_train.to_numpy()
X_test, y_test = X_test.to_numpy(), y_test.to_numpy()

'''
#visualize decision boundary
fig, axes = plt.subplots(1, 2, figsize=(10, 3))

for model, ax in zip([LinearSVC(), LogisticRegression()], axes):
    clf = model.fit(X_train, y_train)
    mglearn.plots.plot_2d_separator(clf, X_train, fill=False, eps=0.5, ax=ax, alpha=.7)
    mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, ax=ax)
    ax.set_title("{}".format(clf.__class__.__name__))
    ax.set_xlabel("Feature 0")
    ax.set_ylabel("Feature 1")
axes[0].legend()

#determining strength of the regularization called C --> a higher values of c corresponds to
#less regularization. in other words using a high value for parameter C, the functions try to
#fit the training set as best as possible, while with low values of parameter C, the model puts
#more emphasis on finding a coefficient vector (w) that is close to zero
mglearn.plots.plot_linear_svc_regularization()
'''

#accuracy = opposite of error --> 1 - accuracy = error
#here C=1 is default --> C=100 equals higher accuracy while C=0.01 equals lower accuracy
logreg = LogisticRegression(C=1).fit(X_train, y_train)
print("Training set error rate: {:.3f}".format(1 - logreg.score(X_train, y_train)))
print("Test set error rate    : {:.3f}".format(1 - logreg.score(X_test, y_test)))

'''
logreg100 = LogisticRegression(C=100).fit(X_train, y_train)
print("Training set score, C=100 : {:.3f}".format(logreg100.score(X_train, y_train)))
print("Test set score, C=100     : {:.3f}".format(logreg100.score(X_test, y_test)))

logreg001 = LogisticRegression(C=0.01).fit(X_train, y_train)
print("Training set score, C=0.01 : {:.3f}".format(logreg001.score(X_train, y_train)))
print("Test set score, C=0.01     : {:.3f}".format(logreg001.score(X_test, y_test)))
'''

print("weights      : {}".format(logreg.coef_))
print("intercept (b): {}".format(logreg.intercept_))


#!/usr/bin/python
# -*- coding: utf8 -*-
from django.contrib import admin
from mainapp.models import *
# Register your models here.
class KeyWordAdmin(admin.ModelAdmin):
	fieldsets = [
		('Словосочетание', {'fields': ['Name']}),
		('Услуга или сырье', {'fields': ['Unit']})
	]

class ArticleAdmin(admin.ModelAdmin):
	fieldsets = [
		('Текст', {'fields': ['Text']}),
		('Словосочетание', {'fields': ['KeyWord']}),
		('Источник', {'fields': ['OwnerUrl']})
	]

class ParagraphAdmin(admin.ModelAdmin):
	fieldsets = [
		('Наименование', {'fields': ['Name']}),
		('картинка', {'fields': ['Image']}),
		('Признак большая картинка', {'fields': ['Large']}),
		('Порядковый номер', {'fields': ['No']}),
		('Ориентация по левому краю', {'fields': ['AlignLeft']}),
		('В алфавитном порядке', {'fields': ['AlphabetOrder']}),
		('Флаг перевозки грузов да/нет', {'fields': ['TransportFlag']})
	]

class UnitAdmin(admin.ModelAdmin):
	fieldsets = [
		('Наименование', {'fields': ['Name']}),
		('Описание', {'fields': ['Description']}),
		('Параграф', {'fields': ['Paragraph']}),
		('Транслит английского', {'fields': ['Eng']}),
		('Закупка кубовыми ёмкостями, бочками', {'fields': ['CanBuyInCubes']}),
		('Доставка в регионы', {'fields': ['DeliveryInRegion']}),
		('Услуги по разгрузке/погрузке', {'fields': ['CanLoadUnload']})
	]

class CustomerAdmin(admin.ModelAdmin):
	fieldsets = [
		('Тип заказчика', {'fields': ['Type']})
	]
class GetInTouchAdmin(admin.ModelAdmin):
	fieldsets = [
		('Имя', {'fields': ['Name']}),
		('Номер телефона', {'fields': ['TelNo']}),
		('э.почта', {'fields': ['Email']}),
		('Текст', {'fields': ['Text']}),
		('Количество (кг)', {'fields': ['Quantity']}),
		('Дата', {'fields': ['DateTime']}),
		('Сырье', {'fields': ['Unit']}),
		('Заказчик', {'fields': ['Customer']}),
		('Дата доставки', {'fields': ['DeliveryDate']})
	]
admin.site.register(Article, ArticleAdmin)
admin.site.register(KeyWord, KeyWordAdmin)
admin.site.register(GetInTouch, GetInTouchAdmin)
admin.site.register(Customer, CustomerAdmin)
admin.site.register(Paragraph, ParagraphAdmin)
admin.site.register(Unit, UnitAdmin)


import time

from community import community_louvain
import torch
from parsor import parameter_parser
from clustering import ClusteringMachine
from clustergcn import ClusterGCNTrainer
from utils import tab_printer, graph_reader, feature_reader, target_reader
import deepdish as dd
import networkx as nx
import numpy as np

def main():
    """
    Parsing command line parameters, reading data, graph decomposition, fitting a ClusterGCN and scoring the model.
    """
    # 载入参数
    args = parameter_parser()
    # 为PyTorch生成随机数设定种子
    torch.manual_seed(args.seed)
    # 将参数以列表格式输出
    tab_printer(args)
    # # 读入图G
    # graph = graph_reader(args.edge_path)
    # # 读入特征矩阵X
    # features = feature_reader(args.features_path)
    # # 读入目标矩阵Y
    # target = target_reader(args.target_path)
    start_time = time.time()
    print('Start load data...')
    train_data = dd.io.load('data/botnet/processed/c2_train.hdf5')['0']
    edge_idx = train_data["edge_index"]
    print(f'Data loaded({time.time()-start_time:.2f} s), start preprocessing...')
    start_time = time.time()
    edges = [[edge_idx[0][i], edge_idx[1][i]] for i, _ in enumerate(edge_idx[0])]
    # # Metis不支持自连接
    # edges = [[edge_idx[0][i], edge_idx[1][i]] for i, _ in enumerate(edge_idx[0]) if
    #          not edge_idx[0][i] == edge_idx[1][i]]
    graph = nx.Graph(edges)
    #print(f'Graph build finished({time.time()-start_time:.2f} s), start partition...')
    #start_time = time.time()
    #groups = community_louvain.best_partition(graph)
    #print(f'Partition finished({time.time() - start_time:.2f} s), start calculating modularity...')
    #start_time = time.time()
    #print(f'num_clusters={len(set(groups.values()))}, modularity={community_louvain.modularity(groups, graph)}, {time.time()-start_time:.2f} s')

    adj_view = graph.adj
    full_adj = [np.array([i[0] for i in adj_view[idx].items()], dtype=int) for idx in range(len(adj_view))]
    print(full_adj[0])
    graph = nx.MultiGraph(edges)

    exit(0)
    num_edges = 0
    for i in range(len(full_adj)):
        num_edges += len(full_adj[i])
    print(graph)
    print(f'num_edges={num_edges}, len(full_adj)={len(full_adj)}, len(full_adj[0])={len(full_adj[0])}')
    features = train_data["x"]
    target = np.reshape(train_data["y"], (-1, 1))
    # 图分解，划分特征和目标
    clustering_machine = ClusteringMachine(args, graph, features, target)
    clustering_machine.decompose()
    # 训练模型
    gcn_trainer = ClusterGCNTrainer(args, clustering_machine)

    gcn_trainer.train()
    # 预测
    gcn_trainer.test()

if __name__ == "__main__":
    main()


import numpy as np # linear algebra
#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from tflite_runtime.interpreter import Interpreter
#import tensorflow as tf

# Load the TFLite model and allocate tensors.
#interpreter = tf.lite.Interpreter(model_path="rw_seq_model.tflite")
interpreter = Interpreter(model_path="rw_seq_model.tflite")
interpreter.allocate_tensors()

# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

# Test the model on random input data.
input_shape = input_details[0]['shape']
print(input_shape)

input_data = np.array([[0.6028,-1.0966,-0.3046,0.1674,-0.5065,1.0156 ]], dtype=np.float32)
print(input_data.shape)

interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()

# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])
print("Walk status:" + str(output_data[0][0]))

input_data_run = np.array([[-1.0739,0.0033,0.1052,2.1021,-0.9863,0.2939]],dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], input_data_run)

interpreter.invoke()

# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])
print("Run status:"+ str(output_data[0][0]))

#print("Predict for run [{:.6%}]".(output_data))


import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from matplotlib.colors import ListedColormap
from sklearn.metrics import confusion_matrix

def plot_confusion_matrix(y_true, predictions, title=None, figsize=(8,8), cmap='icefire'):
    """

    """
    plt.figure(figsize=figsize)
    sns.heatmap(confusion_matrix(y_true, predictions), annot=True, fmt='d', cmap=cmap)

    if title:
        plt.title(title)
    plt.xlabel('Actual Class')
    plt.ylabel('Predicted Class')
    plt.show()

def plot_predictions(y_true, predictions, title=None, figsize=(8,8), save_to=None):
    """
    """
    plt.figure(figsize=figsize)
    plt.scatter(predictions, y_true)

    if title:
        plt.title(title)
    plt.xlabel('Predicted Values')
    plt.ylabel('Actual Values')
    if save_to:
        plt.savefig(save_to)
    plt.show()

def plot_decision_regions(X, y, classifier, resolution=0.02, test_idx=None, xlabel='feature1',
                            ylabel='feature2', title=None, legend=None):
    """
    Conveinience function to visualize the decision boundaries
    for two-dimensional datasets
    """
    # setup marker generator and color map
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface
    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
                            np.arange(x2_min, x2_max, resolution))
    Z = classifier.predict(np.array([xx1.ravel(),xx2.ravel()]).T)
    Z = Z.reshape(xx1.shape)
    plt.contourf(xx1, xx2, Z, alpha=0.3, cmap=cmap)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    # plot class examples
    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(x=X[y == cl, 0],
                    y=X[y == cl, 1],
                    alpha=0.8,
                    c=colors[idx],
                    marker=markers[idx],
                    label=cl,
                    edgecolor='black')
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    if title:
        plt.title(title)
    if legend:
        plt.legend()

    # highlight test examples
    if test_idx:
        # plot all examples
        X_test, y_test = X[test_idx, :], y[test_idx]

        plt.scatter(X_test[:, 0], X_test[:, 1],
                    c='', edgecolor='black', alpha=1.0,
                    linewidth=1, marker='o',
                    s=100, label='test set')
    plt.show()

def plot_linear(X, y, model):
    plt.scatter(X, y, c='steelblue', edgecolor='white', s=70)
    plt.plot(X, model.predict(X), color='black', lw=2)
    plt.show()


import pygame as pg
from pygame.locals import *

tela = pg.display.set_mode((1080,720),pg.RESIZABLE)

perso = pg.image.load('Projetos/proj_turmaA/sprites.png').conve
perso_ani = []
rec_perso = perso.get_rect()
rec_perso.x = 50
rec_perso.y = 500

vila = pg.Surface((300,300))
vila.fill((165,42,42))
rec_vila = vila.get_rect()
rec_vila.x = 600
rec_vila.y = 200

time = pg.time.Clock()

spt_x = 32
spt_y = 32

for line in range(perso.get_height()//spt_y):
    for col in range(perso.get_width()//spt_x):
        x = col * spt_x
        y = line * spt_y
        trans = perso.subsurface(pg.Rect(x, y, spt_x, spt_y))
        perso_ani.append(trans)

while True:

    time.tick(30)

    for event in pg.event.get():
        if event.type == QUIT:
            pg.quit()
        if event.type == KEYDOWN:
            if event.key == K_ESCAPE:
                pg.quit()
        if pg.key.get_pressed()[K_w]:
            rec_perso.y -= 25

        if pg.key.get_pressed()[K_s]:
            rec_perso.y += 25

        if pg.key.get_pressed()[K_a]:
            rec_perso.x -= 25

        if pg.key.get_pressed()[K_d]:
            rec_perso.x += 25
            for i in range(8,12):
                tela.blit(perso_ani[i])

        if rec_perso.colliderect(rec_vila):
            print('colidiu')

        tela.fill((0,0,0))
        tela.blit(vila,rec_vila)
        tela.blit(perso_ani[0],rec_perso)

    pg.display.update()


"""

Analysis of titanic.xml (originally from
https://pythonprogramming.net/static/downloads/machine-learning-data/titanic.xls), which is a spreadsheet of all of the
known people on the HMS Titanic. The spreadsheet is formatted out like this:

pclass - Passenger Class (1 = 1st; 2 = 2nd; 3 = 3rd)
survival - Survival (0 = No; 1 = Yes)
name - Name
sex - Sex
age - Age
sibsp - Number of Siblings/Spouses Aboard
parch - Number of Parents/Children Aboard
ticket - Ticket Number
fare - Passenger Fare (British pound)
cabin - Cabin
embarked - Port of Embarkation (C = Cherbourg; Q = Queenstown; S = Southampton)
boat - Lifeboat number (if they got onto one)
body - Body Identification Number
home.dest - Home/Destination

This program attempts to determine whether a person would live or die given the above traits (minus survival). As well,
which of these traits are most decisive of a person surviving.

"""

import os
import numpy as np
from sklearn.cluster import KMeans
from sklearn import preprocessing
import pandas as pd

df = pd.read_excel(os.getcwd() + "/titanic.xls")
df.drop(["body", "name"], 1, inplace=True)
df.convert_objects(convert_numeric=True)
df.fillna(0, inplace=True)

def handle_non_numerical_data(df):
    columns = df.columns.values

    for column in columns:
        text_digit_vals = {}
        def convert_to_int(val):
            return text_digit_vals[val]

        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1

            df[column] = list(map(convert_to_int, df[column]))

    return df

df = handle_non_numerical_data(df)

X = np.array(df.drop(["survived"], 1).astype(float))
X = preprocessing.scale(X)
y = np.array(df["survived"])

clf = KMeans(n_clusters=2)
clf.fit(X)

correct = 0
for i in range(len(X)):
    predict_me = np.array(X[i].astype(float))
    predict_me = predict_me.reshape(-1, len(predict_me))
    prediction = clf.predict(predict_me)
    if prediction[0] == y[i]:
        correct += 1

print(correct / len(X))


from mako.template import Template
import unittest
import os
from mako.compat import py3k, py26, py25
from mako.util import function_named
import re
from mako.cache import CacheImpl, register_plugin
from nose import SkipTest
import sys

template_base = os.path.join(os.path.dirname(__file__), 'templates')
module_base = os.path.join(template_base, 'modules')

class TemplateTest(unittest.TestCase):

    def _file_template(self, filename, **kw):
        filepath = self._file_path(filename)
        return Template(uri=filename, filename=filepath,
                            module_directory=module_base, **kw)

    def _file_path(self, filename):
        name, ext = os.path.splitext(filename)

        if py3k:
            py3k_path = os.path.join(template_base, name + "_py3k" + ext)
            if os.path.exists(py3k_path):
                return py3k_path

        return os.path.join(template_base, filename)

    def _do_file_test(self, filename, expected, filters=None,
                        unicode_=True, template_args=None, **kw):
        t1 = self._file_template(filename, **kw)
        self._do_test(t1, expected, filters=filters,
                        unicode_=unicode_, template_args=template_args)

    def _do_memory_test(self, source, expected, filters=None,
                        unicode_=True, template_args=None, **kw):
        t1 = Template(text=source, **kw)
        self._do_test(t1, expected, filters=filters,
                        unicode_=unicode_, template_args=template_args)

    def _do_test(self, template, expected, filters=None, template_args=None,
                                unicode_=True):
        if template_args is None:
            template_args = {}
        if unicode_:
            output = template.render_unicode(**template_args)
        else:
            output = template.render(**template_args)

        if filters:
            output = filters(output)
        eq_(output, expected)

def eq_(a, b, msg=None):
    """Assert a == b, with repr messaging on failure."""
    assert a == b, msg or "%r != %r" % (a, b)

def teardown():
    import shutil
    shutil.rmtree(module_base, True)

def assert_raises(except_cls, callable_, *args, **kw):
    try:
        callable_(*args, **kw)
        success = False
    except except_cls:
        success = True

    # assert outside the block so it works for AssertionError too !
    assert success, "Callable did not raise an exception"

def assert_raises_message(except_cls, msg, callable_, *args, **kwargs):
    try:
        callable_(*args, **kwargs)
        assert False, "Callable did not raise an exception"
    except except_cls:
        e = sys.exc_info()[1]
        assert re.search(msg, str(e)), "%r !~ %s" % (msg, e)
        print(str(e))

def skip_if(predicate, reason=None):
    """Skip a test if predicate is true."""
    reason = reason or predicate.__name__

    def decorate(fn):
        fn_name = fn.__name__
        def maybe(*args, **kw):
            if predicate():
                msg = "'%s' skipped: %s" % (
                    fn_name, reason)
                raise SkipTest(msg)
            else:
                return fn(*args, **kw)
        return function_named(maybe, fn_name)
    return decorate

def requires_python_2(fn):
    return skip_if(lambda: py3k, "Requires Python 2.xx")(fn)

def requires_python_26_or_greater(fn):
    return skip_if(lambda: not py26, "Requires Python 2.6 or greater")(fn)

def requires_python_25_or_greater(fn):
    return skip_if(lambda: not py25, "Requires Python 2.5 or greater")(fn)

def requires_pygments_14(fn):
    try:
        import pygments
        version = pygments.__version__
    except:
        version = "0"
    return skip_if(lambda: version < "1.4")(fn)

def requires_no_pygments_exceptions(fn):
    def go(*arg, **kw):
        from mako import exceptions
        exceptions._install_fallback()
        try:
            return fn(*arg, **kw)
        finally:
            exceptions._install_highlighting()
    return function_named(go, fn.__name__)

class PlainCacheImpl(CacheImpl):
    """Simple memory cache impl so that tests which
    use caching can run without beaker.  """

    def __init__(self, cache):
        self.cache = cache
        self.data = {}

    def get_or_create(self, key, creation_function, **kw):
        if key in self.data:
            return self.data[key]
        else:
            self.data[key] = data = creation_function(**kw)
            return data

    def put(self, key, value, **kw):
        self.data[key] = value

    def get(self, key, **kw):
        return self.data[key]

    def invalidate(self, key, **kw):
        del self.data[key]

register_plugin("plain", __name__, "PlainCacheImpl")


#!/usr/bin/env python
from fdfgen import forge_fdf
from pathlib import Path

service_months = ["September", "October", "November", "December", "January", "February", "March", "April", "May", "June", "July", "August"]

fields = {
  "Name": "Jhovana Gutiérrez",
  "Date of birth": "Fecha de nacimiento (programa)",
  "Date immersed": "Fecha de bautismo (programa)",

  "Check Box1": True, # Man
  "Check Box2": True, # Female
  "Check Box3": True, # Other Sheep
  "Check Box4": True, # Anointed
  "Check Box5": True, # Elder
  "Check Box6": True, # Ministerial Servant
  "Check Box7": True, # Regular Pioneer
}

first_year = 2025
for year_index in range(2):
  year_num = year_index + 1
  year_num_underscore = "_2" if year_num == 2 else ""
  fields[f"Service Year{year_num_underscore}"] = first_year + year_index
  for service_month_index in range(12):
    service_month = service_months[service_month_index]
    service_month_num = service_month_index + 1
    fields[f"{year_num}-Place_{service_month_num}"] = year_index * 60 + service_month_index * 5 + 1
    fields[f"{year_num}-Video_{service_month_num}"] = year_index * 60 + service_month_index * 5 + 2
    fields[f"{year_num}-Hours_{service_month_num}"] = year_index * 60 + service_month_index * 5 + 3
    fields[f"{year_num}-RV_{service_month_num}"] = year_index * 60 + service_month_index * 5 + 4
    fields[f"{year_num}-Studies_{service_month_num}"] = year_index * 60 + service_month_index * 5 + 5
    fields[f"Remarks{service_months[service_month_index]}{year_num_underscore}"] = f"Notas para {service_months[service_month_index]} de {first_year + year_index}"

fdf = forge_fdf("",[*fields.items()],[],[],[])

Path("PublisherCard.es.fdf").write_bytes(fdf)


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index),
    url(r'^register$', views.register),
    url(r'^welcome$', views.welcome),
    url(r'^login$', views.login),
    url(r'^logout$', views.logout),
    url(r'^add_a_book$', views.add_a_book),
    url(r'^books/(?P<id>\d+)$', views.book_parser),
    url(r'^favorite/(?P<id>\d+)$', views.favorite),
]


import argparse
from pathlib import Path

import torch
from transformers import (
    AutoTokenizer,
    GenerationConfig,
    PreTrainedModel,
    PreTrainedTokenizer,
)
from utils import FileStreamer

@torch.no_grad()
def endless_generate(
    model: PreTrainedModel,
    tokenizer: PreTrainedTokenizer,
    text: str,
    log_file: str,
    min_new_tokens: int = 1000,
    max_new_tokens: int = 10000,
) -> str:
    streamer = FileStreamer(tokenizer, log_file)
    input_ids = tokenizer.encode(text, return_tensors="pt").to(model.device)
    generated_tokens = model.generate(
        input_ids,
        generation_config=GenerationConfig(
            # use_cache=True is required, the rest can be changed up.
            use_cache=True,
            min_new_tokens=min_new_tokens,
            max_new_tokens=max_new_tokens,
            penalty_alpha=0.6,
            top_k=4,
            pad_token_id=tokenizer.pad_token_id,
            eos_token_id=tokenizer.eos_token_id,
        ),
        streamer=streamer,
    )
    # Decode the final generated text
    return tokenizer.decode(generated_tokens[0], skip_special_tokens=True)

def main():
    parser = argparse.ArgumentParser()

    # Which experiment to run?
    parser.add_argument(
        "--experiment", choices=["attention_sinks", "transformers", "windowed"], default="attention_sinks"
    )

    # Model args
    parser.add_argument("--model_name_or_path", type=str, default="meta-llama/Llama-2-7b-hf")
    parser.add_argument("--revision", type=str, default="main")
    parser.add_argument("--trust_remote_code", action="store_true")

    # Input text
    parser.add_argument("--text", type=str, default="Vaswani et al. (2017) introduced the Transformers")

    # Generation args
    parser.add_argument("--min_new_tokens", type=int, default=1000)
    parser.add_argument("--max_new_tokens", type=int, default=10000)

    # Where to log
    parser.add_argument("--log_file", type=str, default=None)

    # Window size for windowed and attention_sinks
    parser.add_argument("--window_size", type=int, default=512)

    # Attention Sinks-only settings
    # Attention Sink window size is calculated with args.window_size - args.attention_sink_size
    parser.add_argument("--attention_sink_size", type=int, default=4)

    args = parser.parse_args()

    # Initialize the model, either via transformers or via attention_sinks
    if args.experiment == "transformers":
        from transformers import AutoModelForCausalLM
    else:
        from attention_sinks import AutoModelForCausalLM
    kwargs = {}
    if args.experiment == "attention_sinks":
        kwargs = {
            "attention_sink_size": args.attention_sink_size,
            "attention_sink_window_size": args.window_size - args.attention_sink_size,  # default: 1020
        }
    elif args.experiment == "windowed":
        kwargs = {
            "attention_sink_size": 0,
            "attention_sink_window_size": args.window_size,
        }
    model = AutoModelForCausalLM.from_pretrained(
        args.model_name_or_path,
        revision=args.revision,
        trust_remote_code=bool(args.trust_remote_code),
        torch_dtype=torch.float16,
        device_map="auto",
        **kwargs,
    )
    model.eval()
    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, trust_remote_code=bool(args.trust_remote_code))
    tokenizer.pad_token_id = tokenizer.eos_token_id

    log_file = args.log_file or Path("demo") / "endless_logs" / args.experiment / f"{args.model_name_or_path}.txt"
    endless_generate(
        model,
        tokenizer,
        args.text,
        log_file=log_file,
        min_new_tokens=args.min_new_tokens,
        max_new_tokens=args.max_new_tokens,
    )

if __name__ == "__main__":
    main()


from pythautomata.abstract.probabilistic_model import ProbabilisticModel
from pythautomata.base_types.sequence import Sequence
from pythautomata.utilities import pdfa_utils
import math
from sklearn.metrics import ndcg_score
import numpy as np
epsilon = Sequence()

def ndcg_score_avg(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence]):
    suffixes = list()
    suffixes.append(Sequence() + learned_model.terminal_symbol)
    ndcg = list()
    k = math.ceil(len(learned_model.alphabet) / 2)

    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    all_obs1 = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    all_obs2 = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    for i in range(len(all_obs1)):
        obs1 = np.asarray([all_obs1[i]])
        obs2 = np.asarray([all_obs2[i]])
        ndcg_word_score = ndcg_score(obs1, obs2, k=k)
        ndcg.append(ndcg_word_score)

    return np.mean(np.array(ndcg))

def wer_avg(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence]):
    suffixes = list()
    suffixes.append(Sequence() + learned_model.terminal_symbol)
    wer = 0

    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    all_obs1 = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    all_obs2 = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    for i in range(len(all_obs1)):
        obs1 = np.asarray(all_obs1[i])
        obs2 = np.asarray(all_obs2[i])
        max1 = np.argmax(obs1)
        max2 = np.argmax(obs2)
        if max1 != max2:
            wer += 1

    return wer / len(test_sequences)

def log_probability_error(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence]):
    errors = list()
    for seq in test_sequences:
        errors.append(abs(target_model.log_sequence_weight(seq) -
                      learned_model.log_sequence_weight(seq)))
    return np.mean(np.array(errors))

def out_of_tolerance_elements(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence], tolerance):
    suffixes = list()
    errorCount = 0
    suffixes.append(Sequence() + learned_model.terminal_symbol)
    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    all_obs1 = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    all_obs2 = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    for i in range(len(all_obs1)):
        obs1 = np.asarray(all_obs1[i])
        obs2 = np.asarray(all_obs2[i])
        if not pdfa_utils.are_within_tolerance_limit(obs1, obs2, tolerance):
            errorCount += 1

    return errorCount

def out_of_partition_elements(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence], partitions: int):
    suffixes = list()
    errorCount = 0
    suffixes.append(Sequence() + learned_model.terminal_symbol)
    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    all_obs1 = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    all_obs2 = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    for i in range(len(all_obs1)):
        obs1 = np.asarray(all_obs1[i])
        obs2 = np.asarray(all_obs2[i])
        p1 = pdfa_utils.get_quantized_interval_partitions(obs1, partitions)
        p2 = pdfa_utils.get_quantized_interval_partitions(obs2, partitions)
        result = pdfa_utils.compare_quantized_interval_partition(p1, p2)
        if not result:
            errorCount += 1

    return errorCount

def absolute_error_avg(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence]):
    suffixes = list()
    suffixes.append(Sequence() + learned_model.terminal_symbol)

    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    all_obs1 = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    all_obs2 = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    absolute_error_sum = 0
    for i in range(len(all_obs1)):
        obs1 = np.asarray(all_obs1[i])
        obs2 = np.asarray(all_obs2[i])
        absolute_error_sum += np.sum(np.abs(obs1-obs2))

    return absolute_error_sum / len(test_sequences)

def cross_entropy(target_distribution, learned_distribution):
    # Categorical Cross Entropy Metric implemented following Keras implementation
    y_true = target_distribution
    y_pred = learned_distribution

    mask_0 = y_pred == 0
    y_true = y_true + mask_0*1e-7
    y_pred = y_pred + mask_0*1e-7

    mask_1 = y_pred == 1
    y_true = y_true - mask_1*1e-7
    y_pred = y_pred - mask_1*1e-7

    y_pred = y_pred/np.sum(y_pred)
    cross_entropy = - np.sum(y_true*np.log(y_pred))
    return cross_entropy

def mean_cross_entropy(target_model: ProbabilisticModel, learned_model: ProbabilisticModel, test_sequences: list[Sequence]):
    suffixes = list()
    suffixes.append(Sequence() + learned_model.terminal_symbol)
    for symbol in learned_model.alphabet.symbols:
        suffixes.append(Sequence((symbol,)))

    target_obs = target_model.last_token_probabilities_batch(
        test_sequences, suffixes)
    learned_obs = learned_model.last_token_probabilities_batch(
        test_sequences, suffixes)

    ce_sum = 0
    for i in range(len(target_obs)):
        obs1 = np.asarray(target_obs[i])
        obs2 = np.asarray(learned_obs[i])
        ce_sum += cross_entropy(obs1, obs2)

    return ce_sum / len(test_sequences)


import numpy as np
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
import torch
from torch.utils.data import Dataset, DataLoader
from transformers import SamModel, SamProcessor
from PIL import Image
from skimage import transform, io, segmentation
from segment_anything import sam_model_registry, SamPredictor
from segment_anything.utils.transforms import ResizeLongestSide
from torch.nn.utils.rnn import pad_sequence
from torch.nn import functional as F
import re
from utils import *
from experiment_setup import*

# set seeds
torch.manual_seed(231)
np.random.seed(231)

join = os.path.join

sam_checkpoint = "models/sam_vit_b_01ec64.pth"
model_type = "vit_b"
device = "cuda"

sam = sam_model_registry[model_type](checkpoint=sam_checkpoint)
resize_transform = ResizeLongestSide(sam.image_encoder.img_size)
resize_transform = ResizeLongestSide(sam.image_encoder.img_size)

def create_npz_dataset(data):
    data_path = f"dataset/{data}"
    img_names = os.listdir(f"{data_path}/images")
    resize_transform = ResizeLongestSide(sam.image_encoder.img_size)
    for i, img_name in enumerate(img_names):
        print(f"Processing image ... {i}")
        # Since images are in grayscale, all three channels have the same value
        img = io.imread(f"{data_path}/images/{img_name}") # shape (img_width, img_height, 3)
        label = io.imread(f"{data_path}/labels/{img_name}") # shape (img_width, img_height, 1)

        processed_img = prepare_image(img, resize_transform, sam)
        original_size = img.shape[:2]
        with torch.no_grad():
            img_embeddings = sam.image_encoder(preprocess(processed_img.unsqueeze(dim=0)))

        img_num = int(re.findall(r'\d+', img_name)[0])

        np.savez(f'dataset/npz_base/{data}/{img_name}.npz',
                image=processed_img.cpu(),
                label=label,
                original_size=original_size,
                img_embeddings=img_embeddings.cpu(),
                img_num=img_num)

        # if i == 210: break

# create a dataset class to load npz data and return back image embeddings and ground truth
class NpzDataset(Dataset):
    def __init__(self, data_root, exp_config):
        self.data_root = data_root
        self.npz_files = sorted(os.listdir(self.data_root))
        self.npz_data = [np.load(join(self.data_root, f)) for f in self.npz_files]
        self.bbox_size = exp_config.get('bbox_size', None)
        self.num_pts = exp_config.get('num_pts', None)

        # max_length = max(len(d['label']) for d in self.npz_data)
        # # Pad all arrays with zeros using NumPy
        # padded_labels = [np.pad(d['label'], (0, max_length - len(d['label'])), 'constant') for d in self.npz_data]

        self.labels = [data['label'] for data in self.npz_data]
        self.images = [data['image'] for data in self.npz_data]
        self.original_sizes = [data['original_size'] for data in self.npz_data]
        self.embeddings = [data['img_embeddings'] for data in self.npz_data]
        self.img_nums = [data['img_num'] for data in self.npz_data]
        print(f"loaded {len(self.images)} images from {data_root}")

    def __len__(self):
        return len(self.images)

    def __getitem__(self, index):
        image = self.images[index]
        label = self.labels[index]

        # bboxes = torch.tensor(np.array([get_bbox_from_mask(label, self.bbox_size)]))
        original_size = self.original_sizes[index]
        img_embeddings = self.embeddings[index]
        img_num = self.img_nums[index]

        img_info = {
            'image': image,
            # 'point_coords': resize_transform.apply_coords_torch(torch.tensor(point_coords), original_size),
            # 'original_coords': point_coords,
            'original_size': original_size,
            # 'point_labels': torch.tensor(point_labels),
            # 'boxes': resize_transform.apply_boxes_torch(torch.tensor(np.array([bbox])), original_size),
            'img_embeddings': img_embeddings,
            'img_num': img_num
            }

        if self.bbox_size is not None:
            bbox = get_bbox_from_mask(label, self.bbox_size)
            img_info['boxes'] = resize_transform.apply_boxes_torch(torch.tensor(np.array([bbox])), original_size)

        # fg_point_coords, fg_point_labels = get_random_fg(label, self.num_pts)
        # bg_point_coords, bg_point_labels = get_random_bg(label, bbox, 1)
        # point_coords = np.concatenate((fg_point_coords, bg_point_coords))
        # point_labels = np.concatenate((fg_point_labels, bg_point_labels))
        if self.num_pts:
            point_coords, point_labels = get_random_fg(label, self.num_pts)
            img_info['point_coords'] = resize_transform.apply_coords_torch(torch.tensor(point_coords), original_size)
            img_info['original_coords'] = point_coords
            img_info['point_labels'] = torch.tensor(point_labels)

        # return {
        #  'image': image,
        #  'point_coords': resize_transform.apply_coords_torch(torch.tensor(point_coords), original_size),
        #  'original_coords': point_coords,
        #  'original_size': original_size,
        #  'point_labels': torch.tensor(point_labels),
        #  'boxes': resize_transform.apply_boxes_torch(torch.tensor(np.array([bbox])), original_size),
        #  'img_embeddings': img_embeddings,

        #  'img_num': img_num # Apparently dataloader doesn't like strings
        # }
        return img_info

    # def collate_fn(self, data):

    #     # image = self.images[index]
    #     # label = self.labels[index]
    #     # bboxes = torch.tensor(np.array([get_bbox_from_mask(label)]))
    #     # original_size = self.original_sizes[index]

    #     # return {
    #     #  'image': image,
    #     # #  'boxes': resize_transform.apply_boxes_torch(bboxes, original_size),
    #     #  'original_size': original_size
    #     # }

    #     images = [torch.tensor(d['image']) for d in data] #(3)
    #     labels = [d['label'] for d in data]
    #     original_sizes = [d['original_size'] for d in data]

    #     images = pad_sequence(images, batch_first=True) #(4)
    #     # labels = torch.tensor(labels) #(5)

    #     return { #(6)
    #         'image': images,
    #         'boxes': torch.tensor(np.array([get_bbox_from_mask(label)])),
    #         'original_size': original_size
    #         # 'label': labels
    #     }

def prepare_image(image, transform, device):
    image = transform.apply_image(image)
    image = torch.as_tensor(image, device=device.device)
    return image.permute(2, 0, 1).contiguous()

def preprocess(x: torch.Tensor) -> torch.Tensor:
    """Normalize pixel values and pad to a square input."""
    # Normalize colors
    x = (x - sam.pixel_mean) / sam.pixel_std

    # Pad
    h, w = x.shape[-2:]
    padh = sam.image_encoder.img_size - h
    padw = sam.image_encoder.img_size - w
    x = F.pad(x, (0, padw, 0, padh))
    return x

# if __name__ == "__main__":

    # create_npz_dataset('malignant')
    # create_npz_dataset('benign')


# 서로소 집합(disjoint Set) : 공통 원소가 없는 두 집합 = 합치기 찾기 자료구조 (Union Find)
# 서로소 집합 자료구조 - 서로소 부분 집합들로 나눠진 원소들의 데이터를 처리하기 위한 자료구조
# 서로소 집합 자료구조가 지원하는 두 연산
# - 합집합 : 두개 원소가 포함된 집합을 하나의 집합으로 합치는 연산
# - 찾기 : 특정 원소가 속한 집합이 어떤 집합인지 알려주는 연산
# => 합집합의 부모들을 동일하게 지정해주는걸 반복하면 됨 Ex) A(A') & B(B') B'를 A'으로 해줌
# 보통 더큰 루트노드가 더 작은 루트노드를 가리키는것이 관행이므로 이를 따르도록 함
# 서로소 집합 자료구조에선 연결성을 통해 쉽게 집합의 형태를 확인할 수 있음
# 단! 기본적인 형태의 서로소 집합 자료구조에서는 루트 노드에 즉시 접근할 수 없음 (부모테이블을 확인하며 재귀적으로 거슬러 올라가야함)

# 특정 원소가 속한 집합을 찾기
def find_parent(parent, x): # x: 노드번호
    # 루트 노드가 아니라면, 루트 노드를 찾을 때까지 재귀적으로 호출
    if parent[x] != x:
        return find_parent(parent, parent[x])
    return x

# 두 원소가 속한 집합을 합치기
def union_parent(parent, a, b):
    a = find_parent(parent, a)
    b = find_parent(parent, b)
    if a < b:
        parent[b] = a
    else:
        parent[a] = b

# 노드의 개수와 간선(Union 연산)의 개수 입력 받기
v, e = map(int, input().split())
parent = [0] * (v + 1) # 부모 테이블 초기화하기

# 부모 테이블상에서, 부모를 자기 자신으로 초기화
for i in range(1, v + 1):
    parent[i] = i

# Union 연산을 각각 수행
for i in range(e):
    a, b = map(int, input().split())
    union_parent(parent, a, b)

# 각 원소가 속한 집합 출력하기
print('각 원소가 속한 집합: ', end='')
for i in range(1, v + 1):
    print(find_parent(parent, i), end=' ')

print()

# 부모 테이블 내용 출력하기 => 당장 자신의 부모를 담고 있으며 루트와는 관련 없음을 의미
print('부모 테이블: ', end='')
for i in range(1, v + 1):
    print(parent[i], end=' ')

# =================수행시간에서 단점이 있을 수 있음 (합집합 연산이 편향되게 이뤄지는 경우 찾기 함수가 비효율적임)==================

# 찾기 함수의 최적화 방법으로 경로압축
# 찾기 함수를 재귀적으로 호출한 후 부모 테이블 값을 바로 갱신 ==> 부모의 값이 루트가 될 수 있도록 함

# 특정 원소가 속한 집합을 찾기
def find_parent(parent, x):
    # 루트 노드가 아니라면, 루트 노드를 찾을 때까지 재귀적으로 호출
    if parent[x] != x:
        parent[x] = find_parent(parent, parent[x])
    return parent[x]

#============================================================================================================
# 서로소 집합을 활용한 사이클 판별
# 서로소 집합은 무방향 그래프 내에서 사이클 판별할 때 사용 (방향그래프에서 사이클 여부는 DFS를 이용)

# 특정 원소가 속한 집합을 찾기
def find_parent(parent, x):
    # 루트 노드가 아니라면, 루트 노드를 찾을 때까지 재귀적으로 호출
    if parent[x] != x:
        parent[x] = find_parent(parent, parent[x])
    return parent[x]

# 두 원소가 속한 집합을 합치기
def union_parent(parent, a, b):
    a = find_parent(parent, a)
    b = find_parent(parent, b)
    if a < b:
        parent[b] = a
    else:
        parent[a] = b

# 노드의 개수와 간선(Union 연산)의 개수 입력 받기
v, e = map(int, input().split())
parent = [0] * (v + 1) # 부모 테이블 초기화하기

# 부모 테이블상에서, 부모를 자기 자신으로 초기화
for i in range(1, v + 1):
    parent[i] = i

cycle = False # 사이클 발생 여부

for i in range(e):
    a, b = map(int, input().split())
    # 사이클이 발생한 경우 종료
    if find_parent(parent, a) == find_parent(parent, b):
        cycle = True
        break
    # 사이클이 발생하지 않았다면 합집합(Union) 연산 수행
    else:
        union_parent(parent, a, b)

if cycle:
    print("사이클이 발생했습니다.")
else:
    print("사이클이 발생하지 않았습니다.")

#============================================================================================================
# 신장트리 : 그래프에서 모든 노드를 포함하면서 사이클이 존재하지 않는 부분그래프로 모든 노드가 연결되면서 사이클이 존재하지 않는다는 조건은 트리의 조건이기도 함
# 최소 신장 트리 : 최소의 비용으로 구성되는 신장트리 찾기
# 크쿠스칼 알고리즘 => 최소 신장 트리 알고리즘으로 그리디 알고리즘으로 분류됨
# 1. 간선 데이터를 비용에 따라 오름차순 정렬
# 2. 간선 하나씩 확인하며 현 간선이 사이클을 발생시키는지 확인하며 사이클이 발생하지 않는 경우 최소 신장트리에 포함시킴 이를 모든간선에 대해 반복

# 특정 원소가 속한 집합을 찾기
def find_parent(parent, x):
    # 루트 노드가 아니라면, 루트 노드를 찾을 때까지 재귀적으로 호출
    if parent[x] != x:
        parent[x] = find_parent(parent, parent[x])
    return parent[x]

# 두 원소가 속한 집합을 합치기
def union_parent(parent, a, b):
    a = find_parent(parent, a)
    b = find_parent(parent, b)
    if a < b:
        parent[b] = a
    else:
        parent[a] = b

# 노드의 개수와 간선(Union 연산)의 개수 입력 받기
v, e = map(int, input().split())
parent = [0] * (v + 1) # 부모 테이블 초기화하기

# 모든 간선을 담을 리스트와, 최종 비용을 담을 변수
edges = []
result = 0

# 부모 테이블상에서, 부모를 자기 자신으로 초기화
for i in range(1, v + 1):
    parent[i] = i

# 모든 간선에 대한 정보를 입력 받기
for _ in range(e):
    a, b, cost = map(int, input().split())
    # 비용순으로 정렬하기 위해서 튜플의 첫 번째 원소를 비용으로 설정
    edges.append((cost, a, b)) # 튜플을 이용하면 기본적으로 첫번째 원소를 기준으로 정렬함

# 간선을 비용순으로 정렬
edges.sort()

# 간선을 하나씩 확인하며
for edge in edges:
    cost, a, b = edge
    # 사이클이 발생하지 않는 경우에만 집합에 포함
    if find_parent(parent, a) != find_parent(parent, b):
        union_parent(parent, a, b)
        result += cost

print(result)

#============================================================================================================
# 위상정렬 : 사이클이 없는 방향 그래프의 모든 노드를 방향성에 거스르지 않도록 순서대로 나열하는 것 => 큐와 DFS를 이용 가능
# 예로 선수과목을 고려한 학습 순서 설정

# 진입차수 : 특정한 노드로 들어오는 간선의 개수 Indegree
# 진출차수 : 특정 노드에서 나가는 간선의 개수  Outdegree

# 큐 이용시 진입차수가 0인 모든 노드를 큐에 넣고 큐가 빌때까지 다음을 반복
# 큐에서 원소를 꺼내 해당 노드에서 나가는 간선을 그래프에서 제거, 새롭게 진입차수가 0이 된 노드를 큐에 넣기
# 모든 원소를 방문하기 전에 큐가 빈다면 사이클이 존재함 => 정렬 불가

from collections import deque

# 노드의 개수와 간선의 개수를 입력 받기
v, e = map(int, input().split())
# 모든 노드에 대한 진입차수는 0으로 초기화
indegree = [0] * (v + 1)
# 각 노드에 연결된 간선 정보를 담기 위한 연결 리스트 초기화
graph = [[] for i in range(v + 1)]

# 방향 그래프의 모든 간선 정보를 입력 받기
for _ in range(e):
    a, b = map(int, input().split())
    graph[a].append(b) # 정점 A에서 B로 이동 가능
    # 진입 차수를 1 증가
    indegree[b] += 1

# 위상 정렬 함수
def topology_sort():
    result = [] # 알고리즘 수행 결과를 담을 리스트
    q = deque() # 큐 기능을 위한 deque 라이브러리 사용

    # 처음 시작할 때는 진입차수가 0인 노드를 큐에 삽입
    for i in range(1, v + 1):
        if indegree[i] == 0:
            q.append(i)

    # 큐가 빌 때까지 반복
    while q:
        # 큐에서 원소 꺼내기
        now = q.popleft()
        result.append(now)
        # 해당 원소와 연결된 노드들의 진입차수에서 1 빼기
        for i in graph[now]:
            indegree[i] -= 1
            # 새롭게 진입차수가 0이 되는 노드를 큐에 삽입
            if indegree[i] == 0:
                q.append(i)

    # 위상 정렬을 수행한 결과 출력
    for i in result:
        print(i, end=' ')

topology_sort()


from nltk.tokenize import sent_tokenize

def lines(a, b):
    """Return lines in both a and b"""

    list_a = a.split("\n")
    list_b = b.split("\n")

    return list_compare(list_a, list_b)

def sentences(a, b):
    """Return sentences in both a and b"""

    list_a = sent_tokenize(a)
    list_b = sent_tokenize(b)

    return list_compare(list_a, list_b)

def substrings(a, b, n):
    """Return substrings of length n in both a and b"""

    list_a = get_substrings(a, n)
    list_b = get_substrings(b, n)

    return list_compare(list_a, list_b)

def get_substrings(list, length):

    output = []

    temp = len(list) - length + 1

    for i in range(temp):
        output.append(list[i:length + i])

    return output

def list_compare(a, b):

    output = []

    for i in a:
        for j in b:
            if i == j and j not in output:
                output.append(j)

    return output


from flask import Flask, render_template, request
from prometheus_flask_exporter import PrometheusMetrics
from jaeger_client import Config

app = Flask(__name__)

metrics = PrometheusMetrics(app)

metrics.info("frontend_app_info", "Frontend App Info", version="1.0.3")

metrics.register_default(
    metrics.counter(
        'by_path_counter', 'Request count by paths',
        labels={'path': lambda: request.path}
    )
)

by_endpoint_counter = metrics.counter(
    'by_endpoint_counter', 'Request count by endpoint',
    labels={'endpoint': lambda: request.endpoint}
)

config = Config(
    config={
        'sampler': {
            'type': 'const',
            'param': 1,
        },
        'logging': True,
    },
    service_name='frontend'
)

tracer= config.initialize_tracer()

@app.route("/")
@by_endpoint_counter
def homepage():
    with tracer.start_span('html'):
        return render_template("main.html")

if __name__ == "__main__":
    app.run()


from typing import Any, Literal, Tuple, Union

import numpy as np
import torch
from scipy.special import softmax
from transformers import CamembertTokenizer

class ThaiSentimentAnalysis:

    """
    A class to handle the Thai sentiment analysis model.

    Attributes
    ----------
    tokenizer : transformers.CamembertTokenizer
        The tokenizer of the model.
    model : torch.nn.Module
        The thai sentiment analysis model in evaluate mode.
    id2label : dict
        The mapping from the label id to the label name.

    Methods
    -------
    infer(text:str)->Tuple[Union[Literal['negative', 'neutral', 'positive'] ,None, Any]]
        Infer the sentiment of the given text.
    """

    def __init__(self) -> None:
        """Load the tokenizer and model. and seting up the defination of the infer result."""
        self.tokenizer = CamembertTokenizer.from_pretrained(
            './nlp/thai_tokenizer')
        self.model:torch.nn.Module = torch.jit.load('./nlp/model_thai_sentiment.pt')
        self.id2label = {
        0: "negative",
        1: "neutral",
        2: "positive"
        }

    def infer(self, text:str)->Tuple[Union[Literal['negative', 'neutral', 'positive'] ,None, Any]]:
        """Infer the sentiment of the given text."""
        encoded_input = self.tokenizer(text, return_tensors='pt')
        output = self.model(**encoded_input)
        scores = output[0][0].detach().numpy()
        scores = softmax(scores)

        scores = scores[:-1]
        scores = scores[::-1]
        sentiment = self.id2label[np.argmax(scores)]

        return sentiment, scores[np.argmax(scores)]