import wx

from Bittorrent.HttpUtilities import joinUrl, splitUrl, i2pHttpUrlRegexObj
from Utilities import showInfoMessage, showWarningMessage, showErrorMessage
from VirtualListCtrl import PersistentVirtualListCtrl

class TrackerModifyPanel(wx.Panel):
    def __init__(self, parent, trackerInfo, persister, version, **kwargs):
        wx.Panel.__init__(self, parent, **kwargs)

        ##VARS
        self.trackerInfo = [{'groupId':tierIdx, 'groupPos':tierIdx+1, 'groupName':'Group '+str(tierIdx), 'groupTracker':tier} for tierIdx, tier in enumerate(trackerInfo)]
        if len(self.trackerInfo) == 0:
            self.selectedGroup = None
            self.nextGroupId = 0
            self.nextTrackerId = 0
        else:
            self.selectedGroup = 0
            self.nextGroupId = max(tier['groupId'] for tier in self.trackerInfo) + 1
            self.nextTrackerId = max(max(tracker['trackerId'] for tracker in tier['groupTracker']) for tier in self.trackerInfo) + 1

        self.knownTrackerMapper = {u'crstrack.i2p':u'http://mm3zx3besctrx6peq5wzzueil237jdgscuvn5ugwilxrwzyuajja.b32.i2p/tracker/announce.php',
                                   u'tracker2.postman.i2p':u'http://lnQ6yoBTxQuQU8EQ1FlF395ITIQF-HGJxUeFvzETLFnoczNjQvKDbtSB7aHhn853zjVXrJBgwlB9sO57KakBDaJ50lUZgVPhjlI19TgJ-CxyHhHSCeKx5JzURdEW-ucdONMynr-b2zwhsx8VQCJwCEkARvt21YkOyQDaB9IdV8aTAmP~PUJQxRwceaTMn96FcVenwdXqleE16fI8CVFOV18jbJKrhTOYpTtcZKV4l1wNYBDwKgwPx5c0kcrRzFyw5~bjuAKO~GJ5dR7BQsL7AwBoQUS4k1lwoYrG1kOIBeDD3XF8BWb6K3GOOoyjc1umYKpur3G~FxBuqtHAsDRICkEbKUqJ9mPYQlTSujhNxiRIW-oLwMtvayCFci99oX8MvazPS7~97x0Gsm-onEK1Td9nBdmq30OqDxpRtXBimbzkLbR1IKObbg9HvrKs3L-kSyGwTUmHG9rSQSoZEvFMA-S0EXO~o4g21q1oikmxPMhkeVwQ22VHB0-LZJfmLr4SAAAA.i2p/announce.php',
                                   u'tracker.thepiratebay.i2p':u'http://tracker.thepiratebay.i2p/announce',
                                   u'tracker.welterde.i2p':u'http://tracker.welterde.i2p/announce'}

        ##GUI
        panelItems = wx.GridBagSizer(vgap = 0, hgap = 0)

        #splitter
        self.splitter = wx.SplitterWindow(self, style=wx.CLIP_CHILDREN)
        self.splitter.SetMinimumPaneSize(200)

        ##tracker groups

        groupPanel = wx.Panel(self.splitter)
        groupPanelSizer = wx.GridBagSizer(vgap = 0, hgap = 6)

        #group list
        #Syntax: NameOfColumn, NameOfStat, DataType, ColumnWidth
        cols = [('Id', 'groupId', 'int', 75, False),\
                ('Pos', 'groupPos', 'int', 40, True),\
                ('Group', 'groupName', 'native', 100, True)]

        listboxId = wx.NewId()
        self.trackerGroupList = PersistentVirtualListCtrl(persister, 'TorrentModifyDialog-TrackerGroupList-', self._updateGroupPerstData, version,
                                                          cols, self._getTrackerGroupData, rowIdCol='Id', allowSort=False, defaultSortCol='Pos',
                                                          parent=groupPanel, id=listboxId, size=wx.Size(140, -1))
        self.trackerGroupList.SetToolTipString('')
        groupPanelSizer.Add(self.trackerGroupList, (0,1), (6,1), wx.EXPAND | wx.ALL, border = 0)

        #buttons
        buttonAdd = wx.Button(groupPanel, wx.ID_ADD, "")
        groupPanelSizer.Add(buttonAdd, (1,0), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonAdd, wx.ID_ADD, self.OnGroupAdd)

        buttonMoveUp = wx.Button(groupPanel, wx.ID_UP, "")
        groupPanelSizer.Add(buttonMoveUp, (2,0), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonMoveUp, wx.ID_UP, self.OnGroupMoveUp)

        buttonId = wx.NewId()
        buttonMoveDown = wx.Button(groupPanel, wx.ID_DOWN, "")
        groupPanelSizer.Add(buttonMoveDown, (3,0), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonMoveDown, wx.ID_DOWN, self.OnGroupMoveDown)

        buttonId = wx.NewId()
        buttonRemove = wx.Button(groupPanel, wx.ID_DELETE, "")
        groupPanelSizer.Add(buttonRemove, (4,0), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonRemove, wx.ID_DELETE, self.OnGroupRemove)

        #sizer
        groupPanelSizer.AddGrowableCol(1, 1)
        groupPanelSizer.AddGrowableRow(0, 1)
        groupPanelSizer.AddGrowableRow(5, 1)
        groupPanel.SetSizer(groupPanelSizer)

        ##tracker urls

        urlPanel = wx.Panel(self.splitter)
        urlPanelSizer = wx.GridBagSizer(vgap = 0, hgap = 6)

        #url list
        #Syntax: NameOfColumn, NameOfStat, DataType, ColumnWidth
        cols = [('Id', 'trackerId', 'int', 75, False),\
                ('Position', 'tierPos', 'int', 75, True),\
                ('Url', 'trackerUrl', 'native', 385, True),\
                ('A', 'active', 'bool', 20, False),\
                ('Seeds', 'seeds', 'int', 75, False),\
                ('Leeches', 'leeches', 'int', 75, False),\
                ('Downloads', 'downloads', 'int', 75, False),\
                ('Last Announce Result', 'lastAnnounceResult', 'native', 170, False),\
                ('Last Announce Attempt', 'announceTryTime', 'fullTime', 170, False),\
                ('Last Announce Success', 'announceSuccessTime', 'fullTime', 170, False),\
                ('Last Scrape Attempt', 'scrapeTryTime', 'fullTime', 170, False),\
                ('Last Scrape Success', 'scrapeSuccessTime', 'fullTime', 170, False),\
                ('Announce Attempts', 'announceTryCount', 'int', 170, False),\
                ('Announce Successes', 'announceSuccessCount', 'int', 170, False),\
                ('Scrape Attempts', 'scrapeTryCount', 'int', 170, False),\
                ('Scrape Successes', 'scrapeSuccessCount', 'int', 170, False)]

        listboxId = wx.NewId()
        self.trackerUrlList = PersistentVirtualListCtrl(persister, 'TorrentModifyDialog-TrackerUrlList-', self._updateTrackerPerstData, version,
                                                        cols, self._getTrackerUrlData, rowIdCol='Id', allowSort=False, defaultSortCol='Position',
                                                        parent=urlPanel, id=listboxId)
        self.trackerUrlList.SetToolTipString('')
        urlPanelSizer.Add(self.trackerUrlList, (0,0), (7,2), wx.EXPAND)
        wx.EVT_LIST_ITEM_RIGHT_CLICK(self.trackerUrlList, listboxId, self.OnTrackerListRightClick)

        #combo + url box
        boxId = wx.NewId()
        self.trackerName = wx.ComboBox(urlPanel, boxId, size = wx.Size(155, -1),\
                                       choices=sorted(self.knownTrackerMapper.iterkeys()) + ['other'], style=wx.CB_READONLY)
        self.trackerName.SetToolTipString('Either select one of the predefined trackers here or select "other" to enter a different tracker announce url in the text box to the right')
        urlPanelSizer.Add(self.trackerName, (7,0), (1,1), wx.TOP, border = 4)
        wx.EVT_COMBOBOX(self, boxId, self.OnTrackerNameSelect)

        self.trackerAnnounceUrl = wx.TextCtrl(urlPanel, -1, "")
        self.trackerAnnounceUrl.SetToolTipString('The full announce url of the tracker, including the "http://"-prefix.')
        self.trackerAnnounceUrl.Disable()
        urlPanelSizer.Add(self.trackerAnnounceUrl, (7,1), (1,1), wx.EXPAND | wx.FIXED_MINSIZE | wx.TOP, border = 4)

        #buttons
        buttonAdd = wx.Button(urlPanel, wx.ID_ADD, "")
        urlPanelSizer.Add(buttonAdd, (2,2), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonAdd, wx.ID_ADD, self.OnTrackerAdd)

        buttonMoveUp = wx.Button(urlPanel, wx.ID_UP, "")
        urlPanelSizer.Add(buttonMoveUp, (3,2), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonMoveUp, wx.ID_UP, self.OnTrackerMoveUp)

        buttonMoveDown = wx.Button(urlPanel, wx.ID_DOWN, "")
        urlPanelSizer.Add(buttonMoveDown, (4,2), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonMoveDown, wx.ID_DOWN, self.OnTrackerMoveDown)

        buttonId = wx.NewId()
        buttonRemove = wx.Button(urlPanel, wx.ID_DELETE, "")
        urlPanelSizer.Add(buttonRemove, (5,2), (1,1), wx.EXPAND | wx.TOP | wx.BOTTOM, border=7)
        wx.EVT_BUTTON(buttonRemove, wx.ID_DELETE, self.OnTrackerRemove)

        #sizer
        urlPanelSizer.AddGrowableCol(1, 1)
        urlPanelSizer.AddGrowableRow(1, 1)
        urlPanelSizer.AddGrowableRow(6, 1)
        urlPanel.SetSizer(urlPanelSizer)

        ##main sizer
        self.splitter.SplitVertically(groupPanel, urlPanel, 228)
        panelItems.Add(self.splitter, (0,0), (1,1), wx.EXPAND | wx.ALL, border = 2)
        panelItems.AddGrowableCol(0, 1)
        panelItems.AddGrowableRow(0, 1)
        self.SetSizer(panelItems)

        #fill lists
        self.trackerGroupList.dataUpdate()
        if len(self.trackerInfo) > 0:
            self.trackerGroupList.Select(0, on=1)
        self.trackerUrlList.dataUpdate()

        ##EVENTS
        self.Bind(wx.EVT_LIST_ITEM_SELECTED, self.OnSelect, self.trackerGroupList)

    ##internal functions - list persisting

    def _updateGroupPerstData(self, persColData, currentVersion):
        return persColData

    def _updateTrackerPerstData(self, persColData, currentVersion):
        return persColData

    ##internal functions - refresh

    def _getTrackerGroupData(self):
        for i in xrange(0, len(self.trackerInfo)):
            self.trackerInfo[i]['groupPos'] = i+1
        return self.trackerInfo

    def _getTrackerUrlData(self):
        if self.selectedGroup is None:
            data = []
        else:
            data = self.trackerInfo[self.selectedGroup]['groupTracker']
            for i in xrange(0, len(data)):
                data[i]['tierPos'] = i+1
        return data

    ##external functions - tracker info

    def getTrackerInfo(self):
        return [tier['groupTracker'] for tier in self.trackerInfo]

    ##external functions - events - other

    def OnSelect(self, event):
        newGroup = self.trackerGroupList.getRawData('Pos',self.trackerGroupList.GetFirstSelected()) - 1
        if not self.selectedGroup == newGroup:
            self.selectedGroup = newGroup
            self.trackerUrlList.dataUpdate()

    def OnTrackerNameSelect(self, event):
        selectedTracker = self.trackerName.GetValue()
        if not selectedTracker == '':
            #something did get selected
            if selectedTracker in self.knownTrackerMapper:
                self.trackerAnnounceUrl.Disable()
                self.trackerAnnounceUrl.SetValue(self.knownTrackerMapper[selectedTracker])

            elif selectedTracker == u'other':
                if not self.trackerAnnounceUrl.IsEnabled():
                    #currently disabled
                    self.trackerAnnounceUrl.SetValue('http://')
                    self.trackerAnnounceUrl.Enable()

    ## external functions - events - groups

    def OnGroupAdd(self, event):
        #determine position for insert
        groupIdx = self.selectedGroup
        if groupIdx is None:
            groupIdx = 0
        else:
            groupIdx += 1

        #insert new group
        self.trackerInfo.insert(groupIdx, {'groupId':self.nextGroupId, 'groupPos':groupIdx+1,
                                           'groupName':'Group '+str(self.nextGroupId), 'groupTracker':[]})
        self.nextGroupId += 1

        #update list
        self.trackerGroupList.dataUpdate()

        #check if it should be selected
        if self.selectedGroup is None:
            self.selectedGroup = groupIdx
            self.trackerGroupList.Select(groupIdx, on=1)

    def OnGroupRemove(self, event):
        #remove all selected groups
        selectedGroups = self.trackerGroupList.getSelectedRows()
        selectedGroups.reverse()
        for groupIdx in selectedGroups:
            del self.trackerInfo[groupIdx]
            if groupIdx == self.selectedGroup:
                self.selectedGroup = None
        self.trackerGroupList.dataUpdate()
        self.trackerUrlList.dataUpdate()

    def OnGroupMoveUp(self, event):
        #move all selected groups one row up
        selectedGroups = self.trackerGroupList.getSelectedRows()
        for groupIdx in selectedGroups:
            if groupIdx > 0:
                self.trackerInfo.insert(groupIdx-1, self.trackerInfo.pop(groupIdx))
        self.trackerGroupList.dataUpdate()

    def OnGroupMoveDown(self, event):
        #move all selected groups one row down
        selectedGroups = self.trackerGroupList.getSelectedRows()
        selectedGroups.reverse()
        for groupIdx in selectedGroups:
            if groupIdx < len(self.trackerInfo) - 1:
                self.trackerInfo.insert(groupIdx+1, self.trackerInfo.pop(groupIdx))
        self.trackerGroupList.dataUpdate()

    ##external funcs - events - tracker

    def OnTrackerAdd(self, event):
        if self.selectedGroup is None:
            #no group selected
            showErrorMessage(self, 'You need to select a tracker group, before you can add a tracker!')
        else:
            #group selected, go on
            tier = self.trackerInfo[self.selectedGroup]['groupTracker']
            trackerUrl = self.trackerAnnounceUrl.GetValue()
            if i2pHttpUrlRegexObj.match(trackerUrl) is None:
                #invalid url
                showErrorMessage(self, 'The url "%s" is not a valid i2p http url!', trackerUrl)
            else:
                #valid
                trackerPos = self.trackerUrlList.GetFirstSelected()
                if trackerPos == -1:
                    trackerPos = 0
                else:
                    trackerPos += 1

                tier.insert(trackerPos, {'tier':self.selectedGroup + 1,
                                         'tierPos':trackerPos + 1,
                                         'trackerUrl':trackerUrl,
                                         'trackerId':self.nextTrackerId,
                                         'active':False,
                                         'announceTryCount':None,
                                         'announceTryTime':None,
                                         'announceSuccessCount':None,
                                         'announceSuccessTime':None,
                                         'lastAnnounceResult':'None',
                                         'scrapeTryCount':None,
                                         'scrapeTryTime':None,
                                         'scrapeSuccessCount':None,
                                         'scrapeSuccessTime':None,
                                         'seeds':None,
                                         'leeches':None,
                                         'downloads':None})
                self.nextTrackerId += 1

                self.trackerUrlList.dataUpdate()

    def OnTrackerRemove(self, event):
        if self.selectedGroup is not None:
            tier = self.trackerInfo[self.selectedGroup]['groupTracker']
            selectedTrackers = self.trackerUrlList.getSelectedRows()
            selectedTrackers.reverse()
            for trackerIdx in selectedTrackers:
                del tier[trackerIdx]
            self.trackerUrlList.dataUpdate()

    def OnTrackerMoveUp(self, event):
        if self.selectedGroup is not None:
            tier = self.trackerInfo[self.selectedGroup]['groupTracker']
            selectedTrackers = self.trackerUrlList.getSelectedRows()
            for trackerIdx in selectedTrackers:
                if trackerIdx > 0:
                    tier.insert(trackerIdx-1, tier.pop(trackerIdx))
            self.trackerUrlList.dataUpdate()

    def OnTrackerMoveDown(self, event):
        if self.selectedGroup is not None:
            tier = self.trackerInfo[self.selectedGroup]['groupTracker']
            selectedTrackers = self.trackerUrlList.getSelectedRows()
            selectedTrackers.reverse()
            for trackerIdx in selectedTrackers:
                if trackerIdx < len(tier) - 1:
                    tier.insert(trackerIdx+1, tier.pop(trackerIdx))
            self.trackerUrlList.dataUpdate()

    def OnTrackerChangeUrl(self):
        trackerIdx = self.trackerUrlList.GetFirstSelected()
        if trackerIdx == -1:
            #no tracker selected
            showErrorMessage(self, 'You need to select a tracker, before you can change its url!')
        else:
            #a tracker was selected
            tracker = self.trackerInfo[self.selectedGroup]['groupTracker'][trackerIdx]
            diag = wx.TextEntryDialog(self, message='Please enter the new url for this tracker:', caption='Enter url', defaultValue=tracker['trackerUrl'])

            if diag.ShowModal() == wx.ID_OK:
                #user did hit ok
                trackerUrl = diag.GetValue()
                if i2pHttpUrlRegexObj.match(trackerUrl) is None:
                    #invalid url
                    showErrorMessage(self, 'The url "%s" is not a valid i2p http url!', trackerUrl)
                else:
                    #valid
                    tracker['trackerUrl'] = trackerUrl
                    self.trackerUrlList.dataUpdate()

    def OnTrackerMoveToGroup(self):
        trackerIdx = self.trackerUrlList.GetFirstSelected()
        if trackerIdx == -1:
            #no tracker selected
            showErrorMessage(self, 'You need to select a tracker, before you can move it to a different group!')
        else:
            #a tracker was selected
            diag = wx.SingleChoiceDialog(self, message='Please select the new group for this tracker:', caption='Select group', choices=[tier['groupName'] for tier in self.trackerInfo])
            if diag.ShowModal() == wx.ID_OK:
                #user did hit ok
                newGroupIdx = diag.GetSelection()
                if not newGroupIdx == self.selectedGroup:
                    #not the current one
                    tracker = self.trackerInfo[self.selectedGroup]['groupTracker'][trackerIdx]
                    del self.trackerInfo[self.selectedGroup]['groupTracker'][trackerIdx]
                    self.trackerInfo[newGroupIdx]['groupTracker'].append(tracker)

                    #move selection
                    self.trackerGroupList.Select(self.selectedGroup, on=0)
                    self.trackerGroupList.Select(newGroupIdx, on=1)

    def OnTrackerListRightClick(self, event):
        diag = TrackerListOptionsPopup(self)
        self.PopupMenu(diag)

class TrackerListOptionsPopup(wx.Menu):
    def __init__(self, trackerModifyPanel, *args, **kwargs):
        wx.Menu.__init__(self, *args, **kwargs)
        #static
        self.trackerModifyPanel = trackerModifyPanel

        #menu
        id = wx.NewId()
        self.AppendCheckItem(id, 'Change url', 'Change the url of the selected tracker')
        self.Bind(wx.EVT_MENU, self.OnChangeUrl, id=id)

        id = wx.NewId()
        self.AppendCheckItem(id, 'Move to other group', 'Move the selected tracker to another group')
        self.Bind(wx.EVT_MENU, self.OnMoveToGroup, id=id)

    def OnChangeUrl(self, event):
        self.trackerModifyPanel.OnTrackerChangeUrl()

    def OnMoveToGroup(self, event):
        self.trackerModifyPanel.OnTrackerMoveToGroup()

class TrackerModifyDialog(wx.Frame):
    def __init__(self, parent, trackerInfo, trackerSetFunc, persister, version, **kwargs):
        wx.Frame.__init__(self, parent, -1, 'Modify Trackers', size=wx.Size(800, 400),\
                          style = wx.DEFAULT_FRAME_STYLE, **kwargs)

        ##vars
        self.trackerSetFunc = trackerSetFunc

        ##basics
        self.CentreOnScreen()
        frameSizer = wx.BoxSizer(wx.VERTICAL)

        mainPanel = wx.Panel(self)
        mainPanelSizer = wx.BoxSizer(wx.VERTICAL)
        mainPanelItems = wx.GridBagSizer(vgap = 0, hgap = 6)

        ##tracker panel
        self.trackerPanel = TrackerModifyPanel(mainPanel, trackerInfo, persister, version)
        mainPanelItems.Add(self.trackerPanel, (0,0), (1,1), wx.EXPAND | wx.ALL, border = 2)

        ##buttons

        mainPanelButtonItems = wx.GridBagSizer(vgap = 3, hgap = 5)

        #create button
        button = wx.Button(mainPanel, wx.ID_OK, "")
        mainPanelButtonItems.Add(button, (0,0), (1,1), wx.ALIGN_CENTER_HORIZONTAL)
        wx.EVT_BUTTON(button, wx.ID_OK, self.OnOkButton)

        #cancel button
        button = wx.Button(mainPanel, wx.ID_CANCEL, "", style=wx.BU_EXACTFIT)
        mainPanelButtonItems.Add(button, (0,1), (1,1), wx.ALIGN_CENTER_HORIZONTAL)
        wx.EVT_BUTTON(button, wx.ID_CANCEL, self.OnCancelButton)
        self.Bind(wx.EVT_CLOSE, self.OnCancelButton)

        #sizer
        mainPanelButtonItems.AddGrowableCol(0, 1)
        mainPanelButtonItems.AddGrowableCol(1, 1)
        mainPanelItems.Add(mainPanelButtonItems, (1,0), (1,1), wx.EXPAND | wx.ALL, border = 2)

        ##main
        mainPanelItems.AddGrowableCol(0, 1)
        mainPanelItems.AddGrowableRow(0, 1)
        mainPanelSizer.Add(mainPanelItems, 1, wx.EXPAND | wx.ALL, border = 2)
        mainPanel.SetSizer(mainPanelSizer)

        frameSizer.Add(mainPanel, 1, wx.EXPAND | wx.ALL, border = 0)
        self.SetSizer(frameSizer)
        self.Layout()
        self.Show()

    def OnOkButton(self, event):
        self.trackerSetFunc(self.trackerPanel.getTrackerInfo())
        self.Destroy()

    def OnCancelButton(self, event):
        self.Destroy()

if __name__ == '__main__':
    #create GUI
    app = wx.App()
    merk = TrackerModifyDialog(None, [])
    app.MainLoop()


import numpy as np
import ROI
import lane_detect

def process(road):
    # Detecting Lane
    canny_image = lane_detect.LaneDetect(road)

    # Finding region of interest
    height = road.shape[0]
    width = road.shape[1]
    roi_vertices = [
        (0, height),
        (width / 2, height / 2),
        (width, height)
    ]
    masked_image = ROI.ROI(canny_image, np.array([roi_vertices], np.int32))

    # Drawing lines
    lane_image = lane_detect.drawLane(road, masked_image)

    return lane_image


from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from active_learning.core import ActiveLearner, MAL1, MismatchFirstFarthestTraversal, LargestNeighborhood

def fully_supervised(X_train, y_train, X_test, y_test):
    ceiling_classifier = LogisticRegression()
    ceiling_classifier.fit(X_train, y_train)
    y_test_pred = ceiling_classifier.predict(X_test)
    f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
    print ("With all the training dataset labeled, \
    the classification accuracy is  {0}.".format(f1))

def random_sampling(X_train, y_train, X_test, y_test):
    learner = ActiveLearner(X_train, initial_batch_size=200, batch_size=200)
    n_batch = 50
    print("Query strategy: Random sampling...")
    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

def largest_neighborhood(X_train, y_train, X_test, y_test):
    learner = LargestNeighborhood(X_train, batch_size=200, initial_batch_size=1000)
    n_batch = 50

    print("Query strategy: Mismatch first largest neigbourhood...")

    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train_with_propagated_labels()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

def mismatch_first_farthest_traversal(X_train, y_train, X_test, y_test):
    learner = MismatchFirstFarthestTraversal(X_train, initial_batch_size=2000, batch_size=200)
    learner.K = 2000
    n_batch = 50
    print("Query strategy: Mismatch-first farthest-traversal...")
    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        if learner.n_batch == 0:
            learner.batch_size = 2000
        else:
            learner.batch_size = 200

        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train_with_propagated_labels()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

if __name__ == '__main__':
    newsgroups_train = fetch_20newsgroups(subset='train')
    newsgroups_test = fetch_20newsgroups(subset='test')
    vectorizer = TfidfVectorizer()
    classifier = LogisticRegression()

    # The vectors are normalized, thus inner product is equivalent to cosine similarity
    X_train = vectorizer.fit_transform(newsgroups_train.data)
    y_train = newsgroups_train.target
    X_test = vectorizer.transform(newsgroups_test.data)
    y_test = newsgroups_test.target

    # fully_supervised(X_train, y_train, X_test, y_test)
    # The accuracy with all the data labeled should be aroung 81%-82%

    random_sampling(X_train, y_train, X_test, y_test)
    largest_neighborhood(X_train, y_train, X_test, y_test)
    mismatch_first_farthest_traversal(X_train, y_train, X_test, y_test)


import torch
import torch.nn.functional as F
from torch_geometric.nn import GCNConv
from torch_geometric.datasets import Planetoid

# Load the CORA dataset (a benchmark dataset for node classification in a graph)
dataset = Planetoid(root='/tmp/Cora', name='Cora')

class GCN(torch.nn.Module):
    def __init__(self):
        super(GCN, self).__init__()
        # Define GCN layers
        self.conv1 = GCNConv(dataset.num_node_features, 16)
        self.conv2 = GCNConv(16, dataset.num_classes)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        # First GCN layer with ReLU activation
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        # Second GCN layer
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

# Instantiate the model and optimizer
model = GCN()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)

# Training loop
def train():
    model.train()
    optimizer.zero_grad()
    # Use the first graph in the dataset as data
    data = dataset[0]
    # Get the output from the model
    out = model(data)
    # Compute the loss (only for labeled nodes, to simulate semi-supervised setting)
    loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
    # Backpropagate
    loss.backward()
    optimizer.step()
    return loss.item()

# Train for 200 epochs
for epoch in range(200):
    loss = train()
    print(f"Epoch {epoch+1}, Loss: {loss:.4f}")


import json
import uuid
from channels.generic.websocket import AsyncWebsocketConsumer
from .models import Chat, Message
from django.contrib.auth.models import User
from channels.db import database_sync_to_async
from django.db.models import F
from profiles.models import Profile

# docker run -p 6379:6379 -d redis:5

class ChatConsumer(AsyncWebsocketConsumer):
    async def connect(self):
        self.room_name = self.scope['url_route']['kwargs']['room_name']
        self.room_group_name = 'chat_%s' % self.room_name

        # Join room group
        await self.channel_layer.group_add(
            self.room_group_name,
            self.channel_name
        )

        await self.accept()

        await self.channel_layer.group_send(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message': '{} is online'.format(self.scope['user'].username)
            }
        )

    async def disconnect(self, close_code):
        # Leave room group
        await self.channel_layer.group_discard(
            self.room_group_name,
            self.channel_name
        )

        await self.channel_layer.group_send(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message': '{} is offline'.format(self.scope['user'].username)
            }
        )

    # Receive message from WebSocket
    async def receive(self, text_data):
        text_data_json = json.loads(text_data)
        if 'message' in text_data_json:
            message = text_data_json['message']

            # Send message to room group
            await self.channel_layer.group_send(
                self.room_group_name,
                {
                    'type': 'chat_message',
                    'message': message,
                    'author': self.scope['user'].username,
                }
            )
            await database_sync_to_async(self.save_message)(message)

    # Receive message from room group
    async def chat_message(self, event):
        message = event['message']
        if 'author' in event:
            author = event['author']
            # Send message to WebSocket
            await self.send(text_data=json.dumps({
                'message': message,
                'author' : author,
            }))
        else:
            # Send message to WebSocket
            await self.send(text_data=json.dumps({
                'message': message
            }))

    def save_message(self, message):
        Message.objects.create(chat_id = uuid.UUID(self.room_name).hex, author = self.scope['user'], content = message)


"""
下面的文件将会从csv文件中读取读取短信与电话记录，
你将在以后的课程中了解更多有关读取文件的知识。
"""
import csv
with open('texts.csv', 'r') as f:
    reader = csv.reader(f)
    texts = list(reader)

with open('calls.csv', 'r') as f:
    reader = csv.reader(f)
    calls = list(reader)

"""
任务1：
短信和通话记录中一共有多少电话号码？每个号码只统计一次。
输出信息：
"There are <count> different telephone numbers in the records."""

# set up a setlist
allnumber = set()
# find info in dictionary, and the first is sender number, the second is reciever number
for infotext in texts:
    allnumber.add(infotext[0])
    allnumber.add(infotext[1])
for infocall in calls:
    allnumber.add(infocall[0])
    allnumber.add(infocall[1])

print("There are {} different telephone numbers in the records."
      .format(len(allnumber)))


from datetime import datetime
import os
import mne
import json
from db_connections import save_preprocessed_file_location_in_db, save_epochs_file_location_in_db, query_for_index

datatype = 'eeg'
extension = '.fif'
timestamp = datetime.now()
timestamp_str = timestamp.strftime("%d-%b-%Y (%H:%M:%S.%f)")

def set_file_name(subject, level, ctr=None):
    if ctr is not None:
        filename = f'sub_{subject}_{level}_{ctr}_raw.fif'
    else:
        filename = f'sub_{subject}_{level}_raw.fif'
    return filename

def get_path(directory_name):
    path = f'{os.getcwd() }/{directory_name}'
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    return path

def save_preprocessed_data_offline(raw, subject):
    level = 'preprocessed'
    path = get_path('preprocessed_data')
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    raw.save(os.path.join(path, set_file_name(subject, level)), fmt='single', overwrite=True)

async def save_preprocessed_data_online(raw, subject, channel_num):
    level = 'preprocessed'
    path = get_path('preprocessed_data')
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    index = await query_for_index() + 1
    raw.save(os.path.join(path, set_file_name(subject, level, index)), fmt='single', overwrite=False)
    await save_preprocessed_file_location_in_db(index, subject, f'{path}/{set_file_name(subject, level, index)}')

def read_raw_fif(filepath):
    return mne.io.read_raw_fif(filepath, preload=True)

def read_fif_epochs(filepath):
    return mne.read_epochs(filepath, preload=True)

def get_event_dict(subject_id) -> dict:
    file_name = f"{get_path('offline_module_data')}/subject_{subject_id}.json"
    file = open(file_name)
    data = json.load(file)
    return data['Event_Dictionary']

def set_montage():
    montage_path = get_path('montage_folder') + '/AC-64.bvef'
    montage = mne.channels.read_custom_montage(montage_path, head_size=0.085)
    return montage


import random
import urllib.request
from bs4 import BeautifulSoup
import math
import sys
import sqlite3
import os

def get_db():
    con = sqlite3.connect(os.path.join(os.path.dirname(__file__), "ubuntu.db"))
    con.execute("create table if not exists bugs (id text, isok bool, link text, title text)")
    con.commit()
    return con

def db_fetch(bug_number):
    db = get_db()
    crs = db.cursor()
    crs.execute("select id, isok, link, title from bugs where id = ?", [str(bug_number)])
    rows = crs.fetchall()
    db.close()
    if len(rows) == 0:
        return None
    else:
        return rows[0]

def db_store(bug_number, isok, link="", title=""):
    db = get_db()
    db.execute("insert into bugs (id, isok, link, title) values (?, ?, ?, ?)", [str(bug_number), isok, link, title])
    db.commit()
    db.close()

def get_individual(bug_number, known_good=False):
    if known_good:
        # get a bug from our known good list
        db = get_db()
        crs = db.cursor()
        crs.execute("select id, link, title from bugs where isok = 1 order by random() limit 1")
        rows = crs.fetchall()
        id, link, title = rows[0]
        return True, {"title": title, "link": link}
        db.close()
    else:
        val = db_fetch(bug_number)
        if val is None:
            pass  # if we know nothing of this bug, carry on with it
        else:
            id, isok, link, title = val
            if isok == 0:
                print("Bug", bug_number, "known bad")
                return False, None  # bail if we know this one to be a dud
            else:
                # this bug is known to be ok, then return it
                return True, {"title": title, "link": link}
    bug_page = "https://pad.lv/{}".format(bug_number)
    print("trying", bug_page)
    try:
        fp = urllib.request.urlopen(bug_page)
    except:
        db_store(bug_number, False)
        return False, None
    soup = BeautifulSoup(fp.read(), "lxml")
    fp.close()
    project = soup.select("div#watermark a")[0]["href"].split("/")[-1]
    if project != "ubuntu":
        print("Not Ubuntu project:", project)
        db_store(bug_number, False)
        return False, None
    dupe = soup.select("div#bug-is-duplicate .bug-duplicate-details a")
    if dupe:
        override_result = dupe[0]["href"].split("/")[-1]
        print("Duplicate of", override_result)
        db_store(bug_number, False)
        return False, None
    try:
        status = soup.select("div.status-content")[0].text.strip()
    except IndexError:
        print("Bug unreadable")
        db_store(bug_number, False)
        return False, None
    if status in ("New", "Incomplete", "Confirmed"):
        title = soup.select("h1#edit-title")[0].text.strip()
        db_store(bug_number, True, bug_page, title)
        return True, {
            "title": title,
            "link": bug_page
        }
    else:
        print("Not seeking contributions: status", status)
        db_store(bug_number, False)
        return False, None

def get(max_bug, override_result=None, known_good=False):
    # get a random int up to max_bug, but biased towards the more recent
    # https://gamedev.stackexchange.com/a/116875 with min = 0
    while True:
        if override_result:
            result = override_result
            override_result = None
        else:
            result = math.floor(max_bug * pow(random.random(), 0.8))
        isok, bug = get_individual(result, known_good)
        if not isok:
            if max_bug == 0:
                max_bug = get_max_bug_number()
            continue
        return bug

def get_max_bug_number():
    fp = urllib.request.urlopen(
        "https://bugs.launchpad.net/ubuntu/+bugs?orderby=-id&start=0")
    soup = BeautifulSoup(fp.read(), "lxml")
    return int(soup.select("span.bugnumber")[0].text[1:])

if __name__ == "__main__":
    # for testing, run as python3 projects/ubuntu_project.py
    # or python3 projects/ubuntu_project.py 434733
    # or to get a definitely known good bug, python3 projects/ubuntu_project.py -1
    if len(sys.argv) > 1 and sys.argv[1] == "-1":
        print(get(0, None, True))
    elif len(sys.argv) > 1:
        print(get(0, sys.argv[1]))
    else:
        max_bug = get_max_bug_number()
        print(get(max_bug))


# -*- coding: utf-8 -*-

"""
Installs and configures Cinder
"""

import os
import re
import uuid
import logging

from packstack.installer import exceptions
from packstack.installer import processors
from packstack.installer import validators
from packstack.installer.utils import split_hosts

from packstack.installer import basedefs
from packstack.installer import utils

from packstack.modules.shortcuts import get_mq
from packstack.modules.ospluginutils import (getManifestTemplate,
                                             appendManifestFile)

from packstack.installer import exceptions
from packstack.installer import output_messages

#------------------ oVirt installer initialization ------------------

PLUGIN_NAME = "OS-Cinder"
PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')

def initConfig(controller):
    params = [
        {"CMD_OPTION": "cinder-db-passwd",
         "USAGE": "The password to use for the Cinder to access DB",
         "PROMPT": "Enter the password for the Cinder DB access",
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_not_empty],
         "DEFAULT_VALUE": uuid.uuid4().hex[:16],
         "MASK_INPUT": True,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_CINDER_DB_PW",
         "USE_DEFAULT": True,
         "NEED_CONFIRM": True,
         "CONDITION": False},

        {"CMD_OPTION": "cinder-ks-passwd",
         "USAGE": ("The password to use for the Cinder to authenticate with "
                   "Keystone"),
         "PROMPT": "Enter the password for the Cinder Keystone access",
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_not_empty],
         "DEFAULT_VALUE": uuid.uuid4().hex[:16],
         "MASK_INPUT": True,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_CINDER_KS_PW",
         "USE_DEFAULT": True,
         "NEED_CONFIRM": True,
         "CONDITION": False},

        {"CMD_OPTION": "cinder-backend",
         "USAGE": ("The Cinder backend to use, valid options are: lvm, "
                   "gluster, nfs, ceph"),
         "PROMPT": "Enter the Cinder backend to be configured",
         "OPTION_LIST": ["lvm", "gluster", "nfs", "vmdk", "ceph"],
         "VALIDATORS": [validators.validate_options],
         "DEFAULT_VALUE": "lvm",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_CINDER_BACKEND",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    group = {"GROUP_NAME": "CINDER",
             "DESCRIPTION": "Cinder Config parameters",
             "PRE_CONDITION": "CONFIG_CINDER_INSTALL",
             "PRE_CONDITION_MATCH": "y",
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)

    def check_lvm_options(config):
        return (config.get('CONFIG_CINDER_INSTALL', 'n') == 'y' and
                config.get('CONFIG_CINDER_BACKEND', 'lvm') == 'lvm')

    params = [
        {"CMD_OPTION": "cinder-volumes-create",
         "USAGE": ("Create Cinder's volumes group. This should only be done "
                   "for testing on a proof-of-concept installation of Cinder. "
                   "This will create a file-backed volume group and is not "
                   "suitable for production usage."),
         "PROMPT": ("Should Cinder's volumes group be created (for "
                    "proof-of-concept installation)?"),
         "OPTION_LIST": ["y", "n"],
         "VALIDATORS": [validators.validate_options],
         "DEFAULT_VALUE": "y",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_CINDER_VOLUMES_CREATE",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    group = {"GROUP_NAME": "CINDERVOLUMECREATE",
             "DESCRIPTION": "Cinder volume create Config parameters",
             "PRE_CONDITION": check_lvm_options,
             "PRE_CONDITION_MATCH": True,
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)

    def check_lvm_vg_options(config):
        return (config.get('CONFIG_CINDER_INSTALL', 'n') == 'y' and
                config.get('CONFIG_CINDER_BACKEND', 'lvm') == 'lvm' and
                config.get('CONFIG_CINDER_VOLUMES_CREATE', 'y') == 'y')

    params = [
        {"CMD_OPTION": "cinder-volumes-size",
         "USAGE": ("Cinder's volumes group size. Note that actual volume size "
                   "will be extended with 3% more space for VG metadata."),
         "PROMPT": "Enter Cinder's volumes group usable size",
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_not_empty],
         "DEFAULT_VALUE": "20G",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_CINDER_VOLUMES_SIZE",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    group = {"GROUP_NAME": "CINDERVOLUMESIZE",
             "DESCRIPTION": "Cinder volume size Config parameters",
             "PRE_CONDITION": check_lvm_vg_options,
             "PRE_CONDITION_MATCH": True,
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)

    def check_gluster_options(config):
        return (config.get('CONFIG_CINDER_INSTALL', 'n') == 'y' and
                config.get('CONFIG_CINDER_BACKEND', 'lvm') == 'gluster')

    params = [
        {"CMD_OPTION": "cinder-gluster-mounts",
         "USAGE": ("A single or comma separated list of gluster volume shares "
                   "to mount, eg: ip-address:/vol-name, domain:/vol-name "),
         "PROMPT": ("Enter a single or comma separated list of gluster volume "
                    "shares to use with Cinder"),
         "OPTION_LIST": ["^'([\d]{1,3}\.){3}[\d]{1,3}:/.*'",
                         "^'[a-zA-Z0-9][\-\.\w]*:/.*'"],
         "VALIDATORS": [validators.validate_multi_regexp],
         "PROCESSORS": [processors.process_add_quotes_around_values],
         "DEFAULT_VALUE": "",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_CINDER_GLUSTER_MOUNTS",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    group = {"GROUP_NAME": "CINDERGLUSTERMOUNTS",
             "DESCRIPTION": "Cinder gluster Config parameters",
             "PRE_CONDITION": check_gluster_options,
             "PRE_CONDITION_MATCH": True,
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)

    def check_nfs_options(config):
        return (config.get('CONFIG_CINDER_INSTALL', 'n') == 'y' and
                config.get('CONFIG_CINDER_BACKEND', 'lvm') == 'nfs')

    params = [
        {"CMD_OPTION": "cinder-nfs-mounts",
         "USAGE": ("A single or comma seprated list of NFS exports to mount, "
                   "eg: ip-address:/export-name "),
         "PROMPT": ("Enter a single or comma seprated list of NFS exports to "
                    "use with Cinder"),
         "OPTION_LIST": ["^'([\d]{1,3}\.){3}[\d]{1,3}:/.*'"],
         "VALIDATORS": [validators.validate_multi_regexp],
         "PROCESSORS": [processors.process_add_quotes_around_values],
         "DEFAULT_VALUE": "",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_CINDER_NFS_MOUNTS",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    group = {"GROUP_NAME": "CINDERNFSMOUNTS",
             "DESCRIPTION": "Cinder NFS Config parameters",
             "PRE_CONDITION": check_nfs_options,
             "PRE_CONDITION_MATCH": True,
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)

def initSequences(controller):
    if controller.CONF['CONFIG_CINDER_INSTALL'] != 'y':
        return

    cinder_steps = [
        {'title': 'Adding Cinder Keystone manifest entries',
         'functions': [create_keystone_manifest]},
        {'title': 'Adding Cinder Ceph preflight entries',
         'functions': [create_ceph_preflight_manifest]},
        {'title': 'Adding Cinder manifest entries',
         'functions': [create_manifest]}
    ]

    if controller.CONF['CONFIG_CINDER_BACKEND'] == 'lvm':
        cinder_steps.append(
            {'title': 'Checking if the Cinder server has a cinder-volumes vg',
             'functions': [check_cinder_vg]})
    controller.addSequence("Installing OpenStack Cinder", [], [], cinder_steps)

#-------------------------- step functions --------------------------

def check_cinder_vg(config, messages):
    cinders_volume = 'cinder-volumes'
    if config['CONFIG_UNSUPPORTED'] != 'y':
        config['CONFIG_STORAGE_HOST'] = config['CONFIG_CONTROLLER_HOST']

    # Do we have a cinder-volumes vg?
    have_cinders_volume = False
    server = utils.ScriptRunner(config['CONFIG_STORAGE_HOST'])
    server.append('vgdisplay %s' % cinders_volume)
    try:
        server.execute()
        have_cinders_volume = True
    except exceptions.ScriptRuntimeError:
        pass

    # Configure system LVM settings (snapshot_autoextend)
    # Only if /etc/lvm/lvm.conf exists on the system.
    server = utils.ScriptRunner(config['CONFIG_STORAGE_HOST'])
    server.append('[[ -f /etc/lvm/lvm.conf ]] && '
                  'sed -i -r "s/^ *snapshot_autoextend_threshold +=.*/'
                  '    snapshot_autoextend_threshold = 80/" '
                  '/etc/lvm/lvm.conf || '
                  'echo "No /etc/lvm/lvm.conf file found"')
    server.append('[[ -f /etc/lvm/lvm.conf ]] && '
                  'sed -i -r "s/^ *snapshot_autoextend_percent +=.*/'
                  '    snapshot_autoextend_percent = 20/" '
                  '/etc/lvm/lvm.conf || '
                  'echo "No /etc/lvm/lvm.conf file found"')
    try:
        server.execute()
    except exceptions.ScriptRuntimeError:
        logging.info("Warning: Unable to set system LVM settings.")

    if config["CONFIG_CINDER_VOLUMES_CREATE"] != "y":
        if not have_cinders_volume:
            raise exceptions.MissingRequirements("The cinder server should "
                                                 "contain a cinder-volumes "
                                                 "volume group")
    else:
        if have_cinders_volume:
            messages.append(
                output_messages.INFO_CINDER_VOLUMES_EXISTS)
            return

        # TO-DO: This is implemented in cinder::setup_test_volume class.
        #        We should use it instead of this Python code
        server = utils.ScriptRunner(config['CONFIG_STORAGE_HOST'])
        server.append('systemctl')
        try:
            server.execute()
            rst_cmd = 'systemctl restart openstack-cinder-volume.service'
        except exceptions.ScriptRuntimeError:
            rst_cmd = 'service openstack-cinder-volume restart'

        server.clear()
        logging.info("A new cinder volumes group will be created")
        server.append('yum install -y lvm2')

        # Configure system LVM settings (snapshot_autoextend)
        # Just after the lvm2 rpm installation !
        server.append('[[ -f /etc/lvm/lvm.conf ]] && '
                      'sed -i -r "s/^ *snapshot_autoextend_threshold +=.*/'
                      '    snapshot_autoextend_threshold = 80/" '
                      '/etc/lvm/lvm.conf || '
                      'echo "No /etc/lvm/lvm.conf file found"')
        server.append('[[ -f /etc/lvm/lvm.conf ]] && '
                      'sed -i -r "s/^ *snapshot_autoextend_percent +=.*/'
                      '    snapshot_autoextend_percent = 20/" '
                      '/etc/lvm/lvm.conf || '
                      'echo "No /etc/lvm/lvm.conf file found"')

        cinders_volume_path = '/var/lib/cinder'
        server.append('mkdir -p  %s' % cinders_volume_path)
        logging.debug("Volume's path: %s" % cinders_volume_path)

        match = re.match('^(?P<size>\d+)G$',
                         config['CONFIG_CINDER_VOLUMES_SIZE'].strip())
        if not match:
            msg = 'Invalid Cinder volumes VG size.'
            raise exceptions.ParamValidationError(msg)

        cinders_volume_size = int(match.group('size')) * 1024
        cinders_reserve = int(cinders_volume_size * 0.03)

        cinders_volume_size = cinders_volume_size + cinders_reserve
        cinders_volume_path = os.path.join(cinders_volume_path, cinders_volume)
        server.append('dd if=/dev/zero of=%s bs=1 count=0 seek=%sM'
                      % (cinders_volume_path, cinders_volume_size))
        server.append('LOFI=$(losetup --show -f  %s)' % cinders_volume_path)
        server.append('pvcreate $LOFI')
        server.append('vgcreate %s $LOFI' % cinders_volume)

        # Add the loop device on boot
        server.append('grep %(volume)s /etc/rc.d/rc.local || '
                      'echo "losetup -f %(path)s && '
                      'vgchange -a y %(volume)s && '
                      '%(restart_cmd)s" '
                      '>> /etc/rc.d/rc.local' %
                      {'volume': cinders_volume, 'restart_cmd': rst_cmd,
                       'path': cinders_volume_path})
        server.append('grep "#!" /etc/rc.d/rc.local || '
                      'sed -i \'1i#!/bin/sh\' /etc/rc.d/rc.local')
        server.append('chmod +x /etc/rc.d/rc.local')

        # Let's make sure it exists
        server.append('vgdisplay %s' % cinders_volume)

        try:
            server.execute()
        except exceptions.ScriptRuntimeError:
            # Release loop device if cinder's volume creation
            # fails.
            try:
                logging.debug("Release loop device, volume creation failed")
                server = utils.ScriptRunner(config['CONFIG_STORAGE_HOST'])
                server.append('losetup -d $(losetup -j %s | cut -d : -f 1)'
                              % cinders_volume_path)
                server.execute()
            except:
                pass

            raise exceptions.MissingRequirements("Cinder's volume group '%s' "
                                                 "could not be created"
                                                 % cinders_volume)

def create_keystone_manifest(config, messages):
    if config['CONFIG_UNSUPPORTED'] != 'y':
        config['CONFIG_STORAGE_HOST'] = config['CONFIG_CONTROLLER_HOST']

    manifestfile = "%s_keystone.pp" % config['CONFIG_CONTROLLER_HOST']
    manifestdata = getManifestTemplate("keystone_cinder.pp")
    appendManifestFile(manifestfile, manifestdata)

def create_manifest(config, messages):
    if config['CONFIG_UNSUPPORTED'] != 'y':
        config['CONFIG_STORAGE_HOST'] = config['CONFIG_CONTROLLER_HOST']

    manifestdata = getManifestTemplate(get_mq(config, "cinder"))
    manifestfile = "%s_cinder.pp" % config['CONFIG_STORAGE_HOST']
    manifestdata += getManifestTemplate("cinder.pp")

    if config['CONFIG_CINDER_BACKEND'] == "lvm":
        manifestdata += getManifestTemplate("cinder_lvm.pp")
    elif config['CONFIG_CINDER_BACKEND'] == "gluster":
        manifestdata += getManifestTemplate("cinder_gluster.pp")
    elif config['CONFIG_CINDER_BACKEND'] == "nfs":
        manifestdata += getManifestTemplate("cinder_nfs.pp")
    elif config['CONFIG_CINDER_BACKEND'] == "vmdk":
        manifestdata += getManifestTemplate("cinder_vmdk.pp")
    elif config['CONFIG_CINDER_BACKEND'] == "ceph":
        manifestdata += getManifestTemplate("cinder_ceph.pp")
    if config['CONFIG_CEILOMETER_INSTALL'] == 'y':
        manifestdata += getManifestTemplate('cinder_ceilometer.pp')
    if config['CONFIG_SWIFT_INSTALL'] == 'y':
        manifestdata += getManifestTemplate('cinder_backup.pp')

    config['FIREWALL_SERVICE_NAME'] = "cinder"
    config['FIREWALL_PORTS'] = "['3260']"
    config['FIREWALL_CHAIN'] = "INPUT"
    config['FIREWALL_PROTOCOL'] = 'tcp'
    if (config['CONFIG_NOVA_INSTALL'] == 'y' and
            config['CONFIG_VMWARE_BACKEND'] == 'n'):
        for host in split_hosts(config['CONFIG_COMPUTE_HOSTS']):
            config['FIREWALL_ALLOWED'] = "'%s'" % host
            config['FIREWALL_SERVICE_ID'] = "cinder_%s" % host
            manifestdata += getManifestTemplate("firewall.pp")
    else:
        config['FIREWALL_ALLOWED'] = "'ALL'"
        config['FIREWALL_SERVICE_ID'] = "cinder_ALL"
        manifestdata += getManifestTemplate("firewall.pp")
    # cinder API should be open for everyone
    config['FIREWALL_SERVICE_NAME'] = "cinder-api"
    config['FIREWALL_ALLOWED'] = "'ALL'"
    config['FIREWALL_SERVICE_ID'] = "cinder_API"
    config['FIREWALL_PORTS'] = "['8776']"
    manifestdata += getManifestTemplate("firewall.pp")
    appendManifestFile(manifestfile, manifestdata)

def create_ceph_preflight_manifest(config, messages):
    for host in split_hosts(config['CONFIG_CEPH_STORAGE_HOSTS']):
        manifest_file = "%s_ceph_firewall.pp" % host
        # Firewall Rules for dhcp in
        config['FIREWALL_PROTOCOL'] = 'tcp'
        config['FIREWALL_ALLOWED'] = "'ALL'"
        config['FIREWALL_SERVICE_NAME'] = "Ceph monitor: "
        config['FIREWALL_SERVICE_ID'] = "ceph_monitor_%s" % host
        config['FIREWALL_PORTS'] = "['6789']"
        config['FIREWALL_CHAIN'] = "INPUT"
        manifest_data = getManifestTemplate("firewall.pp")
        # Firewall Rules for dhcp out
        config['FIREWALL_PROTOCOL'] = 'tcp'
        config['FIREWALL_ALLOWED'] = "'ALL'"
        config['FIREWALL_SERVICE_NAME'] = "Ceph OSDs"
        config['FIREWALL_SERVICE_ID'] = "ceph_osds_%s" % host
        config['FIREWALL_PORTS'] = "['6800-7100']"
        config['FIREWALL_CHAIN'] = "INPUT"
        manifest_data += getManifestTemplate("firewall.pp")
        appendManifestFile(manifest_file, manifest_data)


import re
from datetime import date, datetime
from enum import Enum
from typing import Optional

from pydantic import BaseModel, PositiveFloat

class AppwriteDocumentPermissions(BaseModel):
    """Appwrite document permissions."""

    read: list[str]
    write: list[str]

class AppwriteDocument(BaseModel):
    """Basic fields for an Appwrite document."""

    id: str
    collection: str
    permissions: AppwriteDocumentPermissions

    def __init__(self, **kwargs) -> None:
        for k in list(kwargs.keys()):
            if isinstance(k, str):
                _k = re.sub("^\\$", "", k)
                kwargs[_k] = kwargs.pop(k)
        super().__init__(**kwargs)

class CoffeeBeanRoast(Enum):
    """Types of roasts."""

    LIGHT = "LIGHT"
    LIGHT_MEDIUM = "LIGHT_MEDIUM"
    MEDIUM = "MEDIUM"
    MEDIUM_DARK = "MEDIUM_DARK"
    DARK = "DARK"
    FRENCH = "FRENCH"
    ESPRESSO = "ESPRESSO"

class CoffeeBag(BaseModel):
    """Coffee bag data."""

    brand: str
    name: str
    mass: PositiveFloat
    start_date: Optional[date] = None
    end_date: Optional[date] = None
    active: bool
    roast: Optional[CoffeeBeanRoast] = None

    def __init__(self, **data) -> None:
        for k in ["start_date", "end_date", "roast"]:
            if data.get(k, None) == "":
                data[k] = None
        super().__init__(**data)

class CoffeeBagDocument(CoffeeBag, AppwriteDocument):
    """Coffee Bag Document."""

class CoffeeCup(BaseModel):
    """Coffee cup data."""

    bag_id: str
    datetime: datetime

class CoffeeCupDocument(CoffeeCup, AppwriteDocument):
    """Coffee Cup Document."""


from maltego_trx.decorator_registry import TransformRegistry

registry = TransformRegistry(
        owner="ACME Corporation",
        author="John Doe <johndoe@acme.com>",
        host_url="https://transforms.acme.com",
        seed_ids=["demo"]
)

# The rest of these attributes are optional

# metadata
registry.version = "0.1"

# global settings
# from maltego_trx.template_dir.settings import api_key_setting
# registry.global_settings = [api_key_setting]

# transform suffix to indicate datasource
# registry.display_name_suffix = " [ACME]"

# reference OAuth settings
# registry.oauth_settings_id = ['github-oauth']


from __future__ import absolute_import, division, print_function

import mesh_tensorflow as mtf
import tensorflow.compat.v1 as tf
from pydantic.dataclasses import dataclass

@dataclass
class OptimizerConfig:
    pass

def clip_by_global_norm(grads, clip_norm):
    """Clip the grads by global norm."""
    global_norm = mtf.sqrt(
        mtf.add_n([mtf.reduce_sum(mtf.square(t)) for t in grads if t is not None])
    )
    multiplier = clip_norm / mtf.maximum(global_norm, clip_norm)
    clipped_grads = [None if t is None else t * multiplier for t in grads]
    return clipped_grads, global_norm

def get_optimizer(loss, params, summary, inp_var_grads=None):
    """Creates and returns an optimizer training op."""

    global_step = tf.train.get_or_create_global_step()  # get global step
    mesh = loss.mesh  # get mesh info from loss
    graph = mesh.graph  # get graph info from mesh

    if inp_var_grads is None:
        var_grads = mtf.gradients(
            [loss], [v.outputs[0] for v in graph.trainable_variables]
        )
    else:
        var_grads = inp_var_grads

    learning_rate = tf.constant(
        value=params["lr"], shape=[], dtype=tf.float32
    )  # grab lr param

    if params["lr_decay"] == "linear":
        learning_rate = tf.train.polynomial_decay(
            learning_rate,
            global_step,
            params["train_steps"],
            end_learning_rate=params["lr"]
            * 0.1,  # decrease to 10% of initial LR according to GPT-3 paper
            power=1.0,
            cycle=False,
        )
    elif params["lr_decay"] == "cosine":
        learning_rate = tf.train.cosine_decay(
            learning_rate,
            global_step,
            params["train_steps"],
            alpha=0.1,  # alpha is min lr value as a fraction of init lr.
        )

    if params["warmup_steps"] > 0:
        global_steps_int = tf.cast(global_step, tf.int32)
        warmup_steps_int = tf.constant(params["warmup_steps"], dtype=tf.int32)

        global_steps_float = tf.cast(global_steps_int, tf.float32)
        warmup_steps_float = tf.cast(warmup_steps_int, tf.float32)

        warmup_percent_done = global_steps_float / warmup_steps_float
        warmup_learning_rate = learning_rate * warmup_percent_done

        is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32)
        learning_rate = (
            1.0 - is_warmup
        ) * learning_rate + is_warmup * warmup_learning_rate

    summary.scalar("lr", learning_rate)

    if params["opt_name"].lower() == "adam":
        optimizer = mtf.optimize.AdamWeightDecayOptimizer(
            learning_rate=learning_rate,
            weight_decay_rate=params["weight_decay"],
            beta_1=params["beta1"],
            beta_2=params["beta2"],
            epsilon=params["epsilon"],
            exclude_from_weight_decay=["norm", "bias"],
        )
    else:
        optimizer = mtf.optimize.AdafactorOptimizer(
            learning_rate=params["lr"],
            decay_rate=params["weight_decay"],
            beta1=params["beta1"],
            epsilon1=params["ada_epsilon1"],
            epsilon2=params["ada_epsilon2"],
        )

    if params["gradient_clipping"] is not None:
        clip_value = mtf.constant(mesh, params["gradient_clipping"], dtype=tf.float32)
        (var_grads, _) = clip_by_global_norm(var_grads, clip_norm=clip_value)

    update_ops = optimizer.apply_grads(var_grads, graph.trainable_variables)
    return learning_rate, update_ops


"""Site Representation."""
# Standard Library
from typing import Tuple
from xml.etree import ElementTree as ET

# Math
from decimal import Decimal

Coordinates = Tuple[Decimal, Decimal]

class Point:
    """Point representation."""

    x: Decimal
    y: Decimal

    def __init__(self, x: Decimal, y: Decimal) -> None:
        """Construct point."""
        self.x = x
        self.y = y

    def __str__(self) -> str:
        """Get String representation."""
        return f"{'{0:.4f}'.format(self.x)}, {'{0:.4f}'.format(self.y)}"

    def __repr__(self) -> str:
        """Get object representation."""
        return self.__str__()

    def __eq__(self, point: "Point") -> str:
        """Get equality between points."""
        return self.x == point.x and self.y == point.y

    def get_tuple(self) -> Coordinates:
        """Get tuple of coordinates (x, y)."""
        return (self.x, self.y)

    def get_xml(self, label, exp) -> str:
        """Get xml representation."""
        point_expression = ET.Element("expression")
        point_expression.set(
            "label", label,
        )
        point_expression.set("exp", exp)
        point_expression.set("type", "point")

        """
        <expression label="G" exp="(p_{x}, p_{y})" type="point" />
        <element type="point" label="G">
            <show object="true" label="true"/>
            <objColor r="0" g="0" b="0" alpha="0"/>
            <layer val="0"/>
            <labelMode val="0"/>
            <pointSize val="5"/>
            <pointStyle val="0"/>
        </element>
        """
        point_element = ET.Element("element")
        point_element.set("type", "point")
        point_element.set("label", label)
        point_show = ET.SubElement(point_element, "show")
        point_show.set("object", "true")
        point_show.set("label", "true")
        point_obj_color = ET.SubElement(point_element, "objColor")
        point_obj_color.set("r", "0")
        point_obj_color.set("g", "0")
        point_obj_color.set("b", "0")
        point_obj_color.set("alpha", "0")
        point_layer = ET.SubElement(point_element, "layer")
        point_layer.set("val", "0")
        point_label_mode = ET.SubElement(point_element, "labelMode")
        point_label_mode.set("val", "0")
        point_size = ET.SubElement(point_element, "pointSize")
        point_size.set("val", "5")
        point_style = ET.SubElement(point_element, "pointStyle")
        point_style.set("val", "0")

        expression_xml = (
            ET.tostring(point_expression, encoding="unicode", method="xml")
            + "\n"
            + ET.tostring(point_element, encoding="unicode", method="xml")
            + "\n"
        )
        return expression_xml


import numbers
from os import fspath
import pathlib
import warnings

import appdirs
import numpy as np
import pyfftw
import qpformat
import qpimage
import tifffile

from . import util

#: Output qpimage.QPSeries sensor data
FILE_SENSOR_DATA_H5 = "sensor_data.h5"
#: Output phase/amplitude TIFF sensor data
FILE_SENSOR_DATA_TIF = "sensor_data.tif"

CACHE_DIR = pathlib.Path(appdirs.user_cache_dir(appname="drymass"))
CACHE_DIR.mkdir(parents=True, exist_ok=True)
PYFFTW_WIDOM_PATH = CACHE_DIR / "pyfftw.wisdom"

if PYFFTW_WIDOM_PATH.exists():
    pyfftw.import_wisdom(
        [w.encode() for w in PYFFTW_WIDOM_PATH.read_text().split("\t")])

def convert(path_in, dir_out, meta_data=None, holo_kw=None, qpretrieve_kw=None,
            bg_data_amp=None, bg_data_pha=None, write_tif=False,
            ret_dataset=False, ret_changed=False, count=None, max_count=None):
    """Convert experimental data to `qpimage.QPSeries` on disk

    Parameters
    ----------
    path_in: str or pathlib.Path
        Input path to file or directory
    dir_out: str or pathlib.Path
        Outuput direcory
    meta_data: dict
        Metadata (see `qpimage.meta.META_KEYS`)
    qpretrieve_kw: dict
        Keyword arguments passed to
        :ref:`qpretrieve <qpretrieve:index>` for
        phase retrieval from interferometric data.
    holo_kw: dict
        This is deprecated, please use `qpretrieve_kw` instead.
    bg_data_amp, bg_data_pha: None, int, or path to file
        The background data for phase and amplitude. One of

        - `None`:
          No background data
        - `int`:
          Image index (starting at 1) of the input data set
          to use as background data
        - `str`, `pathlib.Path`:
          Path to a separate file that is used for background
          correction, relative to the directory in which `path_in`
          is located (`path_in.parent`).
    write_tif: bool
        Export tif images for use with Fiji/ImageJ (tif images
        are only created if they don't already exist or if the
        analysis changed)
    ret_dataset: bool
        Return the qpformat dataset
    ret_changed: bool
        Return True if the dataset changed
    count, max_count: multiprocessing.Value
        Can be used to monitor the progress of the algorithm.
        Initially, the value of `max_count.value` is incremented
        by the total number of steps. At each step, the value
        of `count.value` is incremented.
    """
    path = pathlib.Path(path_in).resolve()
    dout = pathlib.Path(dir_out).resolve()

    h5out = dout / FILE_SENSOR_DATA_H5
    imout = dout / FILE_SENSOR_DATA_TIF

    ds = qpformat.load_data(path=path,
                            meta_data=meta_data,
                            holo_kw=holo_kw,
                            qpretrieve_kw=qpretrieve_kw)

    if not (bg_data_amp is None and bg_data_pha is None):
        # Only set background of data set if there is
        # a background defined.
        bgamp = get_background(bg_data=bg_data_amp,
                               dataset=ds,
                               which="amplitude")
        bgpha = get_background(bg_data=bg_data_pha,
                               dataset=ds,
                               which="phase")
        bg_data = qpimage.QPImage(data=(bgpha, bgamp),
                                  which_data=("phase", "amplitude"))
        ds.set_bg(bg_data)

    if util.is_series_file(h5out):
        with qpimage.QPSeries(h5file=h5out, h5mode="r") as qpsr:
            if (ds.identifier == qpsr.identifier and
                    len(ds) == len(qpsr)):
                # file has same identifier and same number of QPSeries
                create = False
            else:
                create = True
    else:
        create = True

    tif_count = max(1, len(ds)//10)
    if max_count is not None:
        with max_count.get_lock():
            max_count.value += len(ds)
            max_count.value += tif_count

    if create:
        # Write h5 data
        ds.saveh5(h5out, count=count)
    else:
        if count is not None:
            with count.get_lock():
                count.value += len(ds)

    if write_tif and (create or not imout.exists()):
        # Also write tif data
        h5series2tif(h5in=h5out, tifout=imout)
    if count is not None:
        with count.get_lock():
            count.value += tif_count

    ret = [h5out]
    if ret_dataset:
        ret.append(ds)
    if ret_changed:
        ret.append(create)
    if len(ret) == 1:
        ret = ret[0]

    PYFFTW_WIDOM_PATH.write_text(
        "\t".join([w.decode() for w in pyfftw.export_wisdom()]))

    return ret

def get_background(bg_data, dataset, which="phase"):
    """Obtain the background data for a dataset

    Parameters
    ----------
    bg_data: None, int, str, or pathlib.Path
        Represents the background data:

        - None: no background data
        - int: image with index `bg_data - 1` in `dataset` is used
          for background correction
        - str, pathlib.Path: An external file will be used for background
          correction.
    dataset: qpformat.dataset.SeriesData
        The dataset for which the background data is collected.
        No background correction is performed! `dataset` is needed
        for integer `bg_data` and for path-based `bg_data`
        (because of meta data and hologram kwargs).

    Returns
    -------
    bg: 2d np.ndarray
        The background data.
    """
    if which not in ["phase", "amplitude"]:
        raise ValueError("`which` must be 'phase' or 'amplitude'!")
    if bg_data is None:
        bg = np.ones(dataset.get_qpimage(0).shape)
    elif isinstance(bg_data, numbers.Integral):
        if bg_data < 1 or bg_data > len(dataset):
            msg = "Background {} index must be between 1 and {}!"
            raise ValueError(msg.format(which, len(dataset)))
        # indexing in configuration file starts at 1
        if which == "phase":
            bg = dataset.get_qpimage(bg_data - 1).pha
        else:
            bg = dataset.get_qpimage(bg_data - 1).amp
    elif isinstance(bg_data, (str, pathlib.Path)):
        bgpath = pathlib.Path(bg_data)
        dsbg = qpformat.load_data(path=bgpath,
                                  meta_data=dataset.meta_data,
                                  qpretrieve_kw=dataset.qpretrieve_kw)
        if len(dsbg) != 1:
            warnings.warn(
                "Background correction with series data not implemented, "
                + "using first image")
        if which == "phase":
            bg = dsbg.get_qpimage(0).pha
        else:
            bg = dsbg.get_qpimage(0).amp
    else:
        msg = "Unknown type for {} `bg_data`: {}".format(which, bg_data)
        raise ValueError(msg)
    return bg

def h5series2tif(h5in, tifout):
    """Convert a qpimage.QPSeries file to a phase/amplitude TIFF file"""
    with qpimage.QPSeries(h5file=h5in, h5mode="r") as qps, \
            tifffile.TiffWriter(fspath(tifout), imagej=True) as tf:
        for ii in range(len(qps)):
            qpi = qps[ii]
            res = 1 / qpi["pixel size"] * 1e-6  # use µm
            dshape = (1, qpi.shape[0], qpi.shape[1])
            dataa = np.array(qpi.amp, dtype=np.float32).reshape(*dshape)
            datap = np.array(qpi.pha, dtype=np.float32).reshape(*dshape)
            data = np.vstack((datap, dataa))
            tf.save(data=data,
                    resolution=(res, res, None),
                    compress=0,
                    )


from telebot_router import TeleBot
from Services.ClientService import ClientService
from Services.CommandService import CommandService
from Services.MenuService import MenuService
from Services.ProcedureService import ProcedureService
from Services.OrderService import OrderService
from Services.LogFactory import LogFactory

app = TeleBot("Restaurant_Bot")
app.config['api_key'] = '6148540855:AAFNaxGTNxK_H332pSG2eCe4rERBSzO2Q44'

LogFactory.Initialize("bot.log")
logger = LogFactory.logger

ProcedureService.Initialize(app)

@app.route('/help')
def help_handler(dict_message):
    user = ClientService.GetUserByContext(dict_message)
    result = CommandService.get_commands_description(user)
    app.send_message(user.chat_id, result)
    logger.info(f'{user.id} - /help')

@app.route('/menu')
def menu_handler(dict_message):
    user = ClientService.GetUserByContext(dict_message)

    if CommandService.check_commands_permissions('/menu', user):
        result = MenuService.ShowMenu()
        app.send_message(user.chat_id, result)

    logger.info(f'{user.id} - /menu')

@app.route('/myorder')
def my_order_handler(dict_message):
    user = ClientService.GetUserByContext(dict_message)

    if CommandService.check_commands_permissions('/myorder', user):
        result = OrderService.GetUserOrder(user)

        if result != None:
            app.send_message(user.chat_id, f"Ваш заказ: {result}")
        else:
            app.send_message(user.chat_id, "Вы нечего не заказывали.")
    else:
        app.send_message(user.chat_id, "команда не доступна")

    logger.info(f'{user.id} - /myorder')

@app.route('(?!/).+')
def data_handler(dict_message):
    data = dict_message['text']
    user = ClientService.GetUserByContext(dict_message)

    if not ProcedureService.TryContinueProcedure(data, user):
        app.send_message(user.chat_id, "Я вас не понимаю")

    logger.info(f'{user.id} - send data - {data}')

@app.route('/add admin')
def add_admin_handler(dict_message):
    user = ClientService.GetUserByContext(dict_message)

    if CommandService.check_commands_permissions('/add admin', user):
        ProcedureService.StartAddAdminProcedure(user)
    else:
        app.send_message(user.chat_id, "команда не доступна")

    logger.info(f'{user.id} - /addAdmin')

@app.route('/order')
def order_handler(dict_message):
    user = ClientService.GetUserByContext(dict_message)

    if CommandService.check_commands_permissions('/order', user):
        ProcedureService.StartOrderProcedure(user)
    else:
        app.send_message(user.chat_id, "команда не доступна")

    logger.info(f'{user.id} - /order')

# Запуск бота
app.poll(debug=True)


"""
   This script relies on settings in config.toml file
   * extract the list of workflows (vidarr .gsiWorkflow files)
   * extract all gsiOlive files, parse them to extract wf names, their versions and modules
   * find all wf repos, if missing - construct from a gsiWorkflow name
   * bring all info together and format into HTML or tsv. Also, dump a json file
"""
import collections
import tomli
import argparse
import json
import os
from git import Git
import gsiWorkflow
import gsiOlive
import gsiRepository as rP
import htmlRenderer

settings = {}

"""
     We operate with Workflow entries, which are dataclasses

     Research Tag(s): [],
     Clinical Tag(s): []
     Latest Tag:
     Repo:
     Research Olives: []
     Clinical Olives: []
     Code Modules: []
     Data Modules: []
"""

"""
   Load settings file and return a dict with obtained values
"""
def load_config(path):
    try:
        with open(path, "rb") as f:
            toml_dict = tomli.load(f)
            print("Loaded configuration file")
    except tomli.TOMLDecodeError:
        print("Failed to load settings, invalid format")
    return toml_dict

"""
   update local copy of the repo, cd in it checkout main branch and pull
"""
def update_source(path: str, main_branch: str):
    try:
        g = Git(path)
        g.checkout(main_branch)
        g.pull()
    except:
        print("failed to update sources")

"""
    From the list of names, pick the shortest and strip it of all known prefixes
"""
def get_raw_name(names: list) -> str:
    raw_name = min((word for word in names if word), key=len)
    if 'prefixes' in settings.keys():
        for prx in settings['prefixes'].values():
            raw_name = raw_name.replace(prx, "")
            raw_name = raw_name.rstrip("_")
    return raw_name

"""
    Join metadata from olives with gsiWorkflow-derived information, return hash
"""
def join_metadata(olive_data: dict, repo_data: dict, wf: str) -> dict:
    """ Join instance-specific modules with wdl-derived modules, keep things unique """
    merged_data = {}
    d_modules = set()
    c_modules = set()
    try:
        for inst in (settings['instances'].values()):
            if inst in olive_info[wf].keys():
                vetted_olives = [os.path.basename(o) for o in olive_info[wf][inst]['olives']]
                d_modules = olive_data[inst]['data_modules'].union(repo_data['data_modules'])
                c_modules = olive_data[inst]['code_modules'].union(repo_data['code_modules'])
                merged_data[inst] = {'olives': vetted_olives,
                                     'tags': list(olive_data[inst]['tags'])}
        merged_data['latest_tag'] = repo_data['latest_tag']
        merged_data['url'] = repo_data['url']
        merged_data['data_modules'] = list(d_modules)
        merged_data['code_modules'] = list(c_modules)
    except:
        print(f'ERROR: Failed to merge gsiOlive and repo data for {wf}')

    return merged_data

"""
   ====================== Main entrance point to the script =============================
   pass (or not) the following:
   -s settings: path to the TOML file with all settings
   -o output json file
   -p output HTML page

   we have defaults for everything
   at the end, script prints out data as a table and dumps data in a json
"""

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Run parsing script to generate gsiWorkflow status table')
    parser.add_argument('-s', '--settings', help='Settings file in TOML format', required=False, default="config.toml")
    parser.add_argument('-o', '--output-json', help='Output json', required=False, default="gsi_workflows.json")
    parser.add_argument('-p', '--output-page', help='Output page, HTML', required=False, default="gsi_workflows.html")
    args = parser.parse_args()

    settings_path = args.settings
    output_json = args.output_json
    output_page = args.output_page
    ''' A. Load settings and Update local copy of the repo'''
    settings = load_config(settings_path)
    try:
        update_source(settings["repo"]["local_olive_dir"], settings["repo"]["main"])
    except:
        print("ERROR: Failed to update local repo copy from the web")

    ''' B. Load gsiWorkflow names from .vidarrworkflow files without prefixes into a dict keyed by instance '''
    instances = []
    wf_names = {}
    olive_files = {}
    if "instances" in settings.keys():
        for i in settings["instances"].keys():
            instances.append(settings["instances"][i])
        prefixes = []
        if "prefixes" in settings.keys():
            prefixes = settings['prefixes'].values()
        wf_names = gsiWorkflow.extract_wf_names(settings["repo"]["local_olive_dir"], instances, prefixes)
    else:
        print("ERROR: THere are no instances to check, fix your settings")

    ''' C. collect and process olives, extract modules and tags '''
    olive_files = {}
    olive_info = {}
    if 'aliases' in settings.keys():
        olive_files = gsiOlive.collect_olives(settings["repo"]["local_olive_dir"], instances, settings['aliases'])
    else:
        olive_files = gsiOlive.collect_olives(settings["repo"]["local_olive_dir"], instances, {})
    olive_info = gsiOlive.extract_olive_info(olive_files, wf_names)

    vetted_data = {}
    ''' D. If configured, try getting list of repos from github (a dict keyed by gsiWorkflow name with no prefixes)'''
    if 'organization' in settings['repo'].keys() and 'token' in settings['repo'].keys():
        org = settings['repo']['organization']
        token = settings['repo']['token']
        myRepo = rP.githubRepo(org, token)
        repo_list = myRepo.get_repo_list()
        if len(repo_list) == 0:
            print("ERROR: Could not retrieve the list of repositories, check the queue and token are Ok")
        repo_info = {}
        ''' E. use repo list, load vidarrbuild.json and wdl and return a hash with names and modules '''
        for repo in repo_list.keys():
            print(f'Processing repository [{repo}]...')
            try:
                wf_data = myRepo.get_file_content(repo, "vidarrbuild.json")
                wf_info = json.loads(wf_data)
                wf_id = get_raw_name(wf_info['names'])
                if wf_id in olive_info.keys() and len(olive_info[wf_id]) != 0:
                    wf_wdl = myRepo.get_file_content(repo, wf_info['wdl'])
                    wf_wdl_lines = str(wf_wdl, encoding='utf-8').split("\n")
                    wf_latest = myRepo.get_latest_tag(repo)
                    wf_modules = gsiWorkflow.parse_workflow(repo, wf_wdl_lines)
                    repo_info[wf_id] = {'url': repo_list[repo],
                                        'latest_tag': wf_latest,
                                        'data_modules': wf_modules['data_modules'],
                                        'code_modules': wf_modules['code_modules']}
                else:
                    print(f'WARNING: Skipping [{wf_id}] as it is not currently in use...')
            except TypeError:
                print(f'WARNING: Repo [{repo}] Does not have information expected for a gsiWorkflow')
            except:
                print(f'ERROR: Collection of information for [{repo}] failed')

        if len(repo_info) == 0:
            print("ERROR: Information from gsiWorkflow repositories could not be collected")

        ''' F. Join two pieces of information, repo-derived info and gsiOlive-derived info '''
        for wf_id in olive_info.keys():
            if wf_id in repo_info.keys():
                vetted_data[wf_id] = join_metadata(olive_info[wf_id], repo_info[wf_id], wf_id)
            else:
                print(f'ERROR: Was not able to collect data for [{wf_id}]')
    else:
        print("ERROR: Repo credentials are not configured, no update from github is possible")

''' G. Dump the data into json file and generate a HTML page '''
if len(vetted_data) > 0:
    vetted_od = collections.OrderedDict(sorted(vetted_data.items()))
    with open(output_json, "w") as wfj:
        json.dump(vetted_od, wfj)

    html_page = htmlRenderer.convert2page(vetted_od)
    '''Return either HTML table or entire page'''
    with open(output_page, 'w') as op:
        op.write(html_page)
else:
    print("ERROR: Was not able to collect up-to-date information, examine this log and make changes")

# TODO: Fix the following:
"""
   ERROR: Was not able to collect data for [pbcmProjectMedipsPipe]
   ERROR: Was not able to collect data for [crosscheckFingerprintsCollector_fastq]
   ERROR: Was not able to collect data for [umiCollapse_CM]
   ERROR: Was not able to collect data for [dnaSeqQC_MiSeq]
"""

# See PyCharm help at https://www.jetbrains.com/help/pycharm/