"""
This bot runs as FPCBot on wikimedia commons
It implements vote counting and supports bot runs as FPCBot on wikimedia commons
It implements vote counting and supports
moving the finished nomination to the archive.

Programmed by Daniel78 at Commons.

It adds the following commandline arguments:

-test             Perform a testrun against an old log
-close            Close and add result to the nominations
-info             Just print the vote count info about the current nominations
-park             Park closed and verified candidates
-auto             Do not ask before commiting edits to articles
-dry              Do not submit any edits, just print them
-threads          Use threads to speed things up, can't be used in interactive mode
-fpc              Handle the featured candidates (if neither -fpc or -delist is used all candidates are handled)
-delist           Handle the delisting candidates (if neither -fpc or -delist is used all candidates are handled)
-notime           Avoid displaying timestamps in log output
-match pattern    Only operate on candidates matching this pattern
"""

import pywikibot, re, datetime, sys, signal

# Imports needed for threading
import threading, time
from pywikibot import config

class NotImplementedException(Exception):

    """Not implemented."""

class ThreadCheckCandidate(threading.Thread):
    def __init__(self, candidate, check):
        threading.Thread.__init__(self)
        self.candidate = candidate
        self.check = check

    def run(self):
        self.check(self.candidate)

class Candidate:
    """
    This is one picture candidate

    This class just serves as base for the DelistCandidate and FPCandidate classes
    """

    def __init__(
        self,
        page,
        ProR,
        ConR,
        NeuR,
        ProString,
        ConString,
        ReviewedR,
        CountedR,
        VerifiedR,
    ):
        """Page is a pywikibot.Page object ."""
        # Later perhaps this can be cleaned up by letting the subclasses keep the variables
        self.page = page
        self._pro = 0
        self._con = 0
        self._neu = 0
        self._proR = ProR  # Regexp for positive votes
        self._conR = ConR  # Regexp for negative votes
        self._neuR = NeuR  # Regexp for neutral  votes
        self._proString = ProString
        self._conString = ConString
        self._ReviewedR = ReviewedR
        self._CountedR = CountedR
        self._VerifiedR = VerifiedR
        self._votesCounted = False
        self._daysOld = -1
        self._daysSinceLastEdit = -1
        self._creationTime = None
        self._imgCount = None
        self._fileName = None
        self._alternative = None
        self._listPageName = None

    def printAllInfo(self):
        """
        Console output of all information sought after
        """
        try:
            self.countVotes()
            out(
                "%s: S:%02d O:%02d N:%02d D:%02d De:%02d Se:%d Im:%02d W:%s (%s)"
                % (
                    self.cutTitle(),
                    self._pro,
                    self._con,
                    self._neu,
                    self.daysOld(),
                    self.daysSinceLastEdit(),
                    self.sectionCount(),
                    self.imageCount(),
                    self.isWithdrawn(),
                    self.statusString(),
                )
            )
        except pywikibot.exceptions.NoPageError:
            out("%s: -- No such page -- " % self.cutTitle(), color="lightred")

    def nominator(self, link=True):
        """Return the link to the user that nominated this candidate."""
        history = self.page.revisions(reverse=True, total=1)
        for data in history:
            username = data.user
        if not history:
            return "Unknown"
        if link:
            return "[[User:%s|%s]]" % (username, username)
        else:
            return username

    def creator(self):
        """Return the link to the user that created the image, Not implemented yet."""
        pass

    def isSet(self):
        """Check if the nomination page has "/[Ss]et/" in it, if yes it must be a set nomination."""
        if re.search(r"/[Ss]et", self.page.title()):
            return True
        else:
            return False

    def setFiles(self):
        """Try to return list of all files in a set, files in the last gallery in the nomination page."""
        m = re.search(r"<gallery([^\]]*)</gallery>", self.page.get(get_redirect=True))
        text_inside_gallery = m.group(1)
        filesList = []
        for line in text_inside_gallery.splitlines():
            if line.startswith("File:"):
                files = re.sub(r"\|.*", "", line)
                filesList.append(files)
        return filesList

    def findGalleryOfFile(self):
        """Try to find Gallery in the nomination page to make closing users life easier."""
        text = self.page.get(get_redirect=True)
        RegexGallery = re.compile(
            r"(?:.*)Gallery(?:.*)(?:\s.*)\[\[Commons\:Featured[_ ]pictures\/([^\]]{1,180})"
        )
        matches = RegexGallery.finditer(text)
        for m in matches:
            Gallery = m.group(1)
        try:
            Gallery
        except:
            Gallery = ""

        return Gallery

    def countVotes(self):
        """
        Counts all the votes for this nomination
        and subtracts eventual striked out votes
        """

        if self._votesCounted:
            return

        text = self.page.get(get_redirect=True)
        if text:
            text = filter_content(text)

            self._pro = len(re.findall(self._proR, text))
            self._con = len(re.findall(self._conR, text))
            self._neu = len(re.findall(self._neuR, text))
        else:
            out("Warning - %s has no content" % self.page, color="lightred")

        self._votesCounted = True

    def isWithdrawn(self):
        """Withdrawn nominations should not be counted."""
        text = self.page.get(get_redirect=True)
        text = filter_content(text)
        withdrawn = len(re.findall(WithdrawnR, text))
        return withdrawn > 0

    def isFPX(self):
        """Page marked with FPX template."""
        return len(re.findall(FpxR, self.page.get(get_redirect=True)))

    def rulesOfFifthDay(self):
        """Check if any of the rules of the fifth day can be applied."""
        if self.daysOld() < 5:
            return False

        self.countVotes()

        # First rule of the fifth day
        if self._pro <= 1:
            return True

        # Second rule of the fifth day
        if self._pro >= 10 and self._con == 0:
            return True

    def closePage(self):
        """
        Will add the voting results to the page if it is finished.
        If it was, True is returned else False
        """

        # First make a check that the page actually exist:
        if not self.page.exists():
            out('"%s" no such page?!' % self.cutTitle())
            return

        if (self.isWithdrawn() or self.isFPX()) and self.imageCount() <= 1:
            # Will close withdrawn nominations if there are more than one
            # full day since the last edit

            why = "withdrawn" if self.isWithdrawn() else "FPXed"

            oldEnough = self.daysSinceLastEdit() > 0
            out(
                '"%s" %s %s'
                % (
                    self.cutTitle(),
                    why,
                    "closing" if oldEnough else "but waiting a day",
                )
            )

            if not oldEnough:
                return False

            self.moveToLog(why)
            return True

        # We skip rule of the fifth day if we have several alternatives
        fifthDay = False if self.imageCount() > 1 else self.rulesOfFifthDay()

        if not fifthDay and not self.isDone():
            out('"%s" is still active, ignoring' % self.cutTitle())
            return False

        old_text = self.page.get(get_redirect=True)
        if not old_text:
            out("Warning - %s has no content" % self.page, color="lightred")
            return False

        if re.search(r"{{\s*FPC-closed-ignored.*}}", old_text):
            out('"%s" is marked as ignored, so ignoring' % self.cutTitle())
            return False

        if re.search(self._CountedR, old_text):
            out('"%s" needs review, ignoring' % self.cutTitle())
            return False

        if re.search(self._ReviewedR, old_text):
            out('"%s" already closed and reviewed, ignoring' % self.cutTitle())
            return False

        if self.imageCount() <= 1:
            self.countVotes()

        result = self.getResultString()

        new_text = old_text + result

        # Add the featured status to the header
        if self.imageCount() <= 1:
            new_text = self.fixHeader(new_text)

        self.commit(
            old_text,
            new_text,
            self.page,
            self.getCloseCommitComment()
            + (" (FifthDay=%s)" % ("yes" if fifthDay else "no")),
        )

        return True

    def fixHeader(self, text, value=None):
        """
        Will append the featured status to the header of the candidate
        Will return the new text
        @param value If specified ("yes" or "no" string will be based on it, otherwise isPassed() is used)
        """

        # Check if they are alredy there
        if re.match(r"===.*(%s|%s)===" % (self._proString, self._conString), text):
            return text

        status = ""

        if value:
            if value == "yes":
                status = ", %s" % self._proString
            elif value == "no":
                status = ", %s" % self._conString

        if len(status) < 1:
            status = (
                ", %s" % self._proString
                if self.isPassed()
                else ", %s" % self._conString
            )

        return re.sub(r"(===.*)(===)", r"\1%s\2" % status, text, 1)

    def getResultString(self):
        """Must be implemented by the subclasses (Text to add to closed pages)."""
        raise NotImplementedException()

    def getCloseCommitComment(self):
        """Must be implemened by the subclasses (Commit comment for closed pages)."""
        raise NotImplementedException()

    def creationTime(self):
        """
        Find the time that this candidate was created
        If we can't find the creation date, for example due to
        the page not existing we return now() such that we
        will ignore this nomination as too young.
        """
        if self._creationTime:
            return self._creationTime

        history = self.page.revisions(reverse=True, total=1)

        if not history:
            out(
                "Could not retrieve history for '%s', returning now()"
                % self.page.title()
            )
            return datetime.datetime.now()

        for data in history:
            self._creationTime = data["timestamp"]

        # print "C:" + self._creationTime.isoformat()
        # print "N:" + datetime.datetime.utcnow().isoformat()
        return self._creationTime

    def statusString(self):
        """Short status string about the candidate."""
        if self.isIgnored():
            return "Ignored"
        elif self.isWithdrawn():
            return "Withdrawn"
        elif not self.isDone():
            return "Active"
        else:
            return self._proString if self.isPassed() else self._conString

    def daysOld(self):
        """Find the number of days this nomination has existed."""
        if self._daysOld != -1:
            return self._daysOld

        delta = datetime.datetime.utcnow() - self.creationTime()
        self._daysOld = delta.days
        return self._daysOld

    def daysSinceLastEdit(self):
        """
        Number of whole days since last edit

        If the value can not be found -1 is returned
        """
        if self._daysSinceLastEdit != -1:
            return self._daysSinceLastEdit

        try:
            lastEdit = datetime.datetime.strptime(
                str(self.page.editTime()), "%Y-%m-%dT%H:%M:%SZ"
            )
        except:
            return -1

        delta = datetime.datetime.utcnow() - lastEdit
        self._daysSinceLastEdit = delta.days
        return self._daysSinceLastEdit

    def isDone(self):
        """
        Checks if a nomination can be closed
        """
        return self.daysOld() >= 9

    def isPassed(self):
        """
        Find if an image can be featured.
        Does not check the age, it needs to be
        checked using isDone()
        """

        if self.isWithdrawn():
            return False

        if not self._votesCounted:
            self.countVotes()

        return self._pro >= 7 and (self._pro >= 2 * self._con)

    def isIgnored(self):
        """Some nominations currently require manual check."""
        return self.imageCount() > 1

    def sectionCount(self):
        """Count the number of sections in this candidate."""
        text = self.page.get(get_redirect=True)
        return len(re.findall(SectionR, text))

    def imageCount(self):
        """
        Count the number of images that are displayed

        Does not count images that are below a certain threshold
        as they probably are just inline icons and not separate
        edits of this candidate.
        """
        if self._imgCount:
            return self._imgCount

        text = self.page.get(get_redirect=True)

        matches = []
        for m in re.finditer(ImagesR, text):
            matches.append(m)

        count = len(matches)

        if count >= 2:
            # We have several images, check if they are too small to be counted
            for img in matches:

                if re.search(ImagesThumbR, img.group(0)):
                    count -= 1
                else:
                    s = re.search(ImagesSizeR, img.group(0))
                    if s and (int(s.group(1)) <= 150):
                        count -= 1

        self._imgCount = count
        return count

    def existingResult(self):
        """
        Will scan this nomination and check whether it has
        already been closed, and if so parses for the existing
        result.
        The return value is a list of tuples, and normally
        there should only be one such tuple. The tuple
        contains four values:
        support,oppose,neutral,(featured|not featured)
        """
        text = self.page.get(get_redirect=True)
        return re.findall(PreviousResultR, text)

    def compareResultToCount(self):
        """
        If there is an existing result we will compare
        it to a new vote count made by this bot and
        see if they match. This is for testing purposes
        of the bot and to find any incorrect old results.
        """
        res = self.existingResult()

        if self.isWithdrawn():
            out("%s: (ignoring, was withdrawn)" % self.cutTitle())
            return

        elif self.isFPX():
            out("%s: (ignoring, was FPXed)" % self.cutTitle())
            return

        elif not res:
            out("%s: (ignoring, has no results)" % self.cutTitle())
            return

        elif len(res) > 1:
            out("%s: (ignoring, has several results)" % self.cutTitle())
            return

        # We have one result, so make a vote count and compare
        old_res = res[0]
        was_featured = old_res[3] == "featured"
        ws = int(old_res[0])
        wo = int(old_res[1])
        wn = int(old_res[2])
        self.countVotes()

        if (
            self._pro == ws
            and self._con == wo
            and self._neu == wn
            and was_featured == self.isPassed()
        ):
            status = "OK"
        else:
            status = "FAIL"

        # List info to console
        out(
            "%s: S%02d/%02d O:%02d/%02d N%02d/%02d F%d/%d (%s)"
            % (
                self.cutTitle(),
                self._pro,
                ws,
                self._con,
                wo,
                self._neu,
                wn,
                self.isPassed(),
                was_featured,
                status,
            )
        )

    def cutTitle(self):
        """Returns a fixed width title."""
        return re.sub(PrefixR, "", self.page.title())[0:50].ljust(50)

    def cleanTitle(self, keepExtension=False):
        """
        Returns a title string without prefix and extension
        Note that this always operates on the original title and that
        a possible change by the alternative parameter is not considered,
        but maybe it should be ?
        """
        noprefix = re.sub(PrefixR, "", self.page.title())
        if keepExtension:
            return noprefix
        else:
            return re.sub(r"\.\w{1,3}$\s*", "", noprefix)

    def fileName(self, alternative=True):
        """
        Return only the filename of this candidate
        This is first based on the title of the page but if that page is not found
        then the first image link in the page is used.
        Will return the new file name if moved.
        @param alternative if false disregard any alternative and return the real filename
        """
        # The regexp here also removes any possible crap between the prefix
        # and the actual start of the filename.
        if alternative and self._alternative:
            return self._alternative

        if self._fileName:
            return self._fileName

        self._fileName = re.sub(
            "(%s.*?)([Ff]ile|[Ii]mage)" % candPrefix, r"\2", self.page.title()
        )

        if not pywikibot.Page(G_Site, self._fileName).exists():
            match = re.search(ImagesR, self.page.get(get_redirect=True))
            if match:
                self._fileName = match.group(1)

        # Check if file was moved after nomination
        page = pywikibot.Page(G_Site, self._fileName)
        if page.isRedirectPage():
            self._fileName = page.getRedirectTarget().title()

        return self._fileName

    def addToFeaturedList(self, gallery):
        """
        Will add this page to the list of featured images.
        This uses just the base of the gallery, like 'Animals'.
        Should only be called on closed and verified candidates

        This is ==STEP 1== of the parking procedure

        @param gallery The categorization gallery
        """
        if self.isSet():
            file = (self.setFiles())[0]  # Add the first file from gallery.
        else:
            file = self.fileName()

        listpage = "Commons:Featured pictures, list"
        page = pywikibot.Page(G_Site, listpage)
        old_text = page.get(get_redirect=True)

        # First check if we are already on the page,
        # in that case skip. Can happen if the process
        # have been previously interrupted.
        if re.search(wikipattern(file), old_text):
            out(
                "Skipping addToFeaturedList for '%s', page already listed."
                % self.cleanTitle(),
                color="lightred",
            )
            return

        # This function first needs to find the gallery
        # then inside the gallery tags remove the last line and
        # add this candidate to the top
        # Thanks KODOS for a nice regexp gui
        # This adds ourself first in the list of length 4 and removes the last
        # all in the chosen gallery
        out("Looking for gallery: '%s'" % wikipattern(gallery))
        ListPageR = re.compile(
            r"(^==\s*{{{\s*\d+\s*\|%s\s*}}}\s*==\s*<gallery.*>\s*)(.*\s*)(.*\s*.*\s*)(.*\s*)(</gallery>)"
            % wikipattern(gallery),
            re.MULTILINE,
        )
        new_text = re.sub(ListPageR, r"\1%s\n\2\3\5" % file, old_text)
        self.commit(old_text, new_text, page, "Added [[%s]]" % file)

    def addToCategorizedFeaturedList(self, gallery):
        """
        Adds the candidate to the gallery of
        pictures. This is the full gallery with
        the section in that particular page.

        This is ==STEP 2== of the parking procedure

        @param gallery The categorization gallery
        If it's a set, will add all file from the list,
        else just one if single nomination.
        """
        if self.isSet():
            files = self.setFiles()
        else:
            files = []
            files.append(self.fileName())
        for file in files:
            gallery_full_path = "Commons:Featured pictures/" + re.sub(
                r"#.*", "", gallery
            )
            page = pywikibot.Page(G_Site, gallery_full_path)
            old_text = page.get(get_redirect=True)
            section_R = r"#(.*)"
            m = re.search(section_R, gallery)
            try:
                section = m.group(1)
            except AttributeError:
                section = None

            if section != None:

                # Trying to generate a regex for finding the section in a gallery if specified in nomination
                # First we are escaping all parentheses, as they are used in regex
                # Replacement of all uunderscore with \s , some users just copy the url
                # Replacing all \s with \s(?:\s*|)\s, user have linked the section to categories. Why ? To make our lives harder

                section = (
                    section.replace(")", "\)")
                    .replace("(", "\(")
                    .replace("_", " ")
                    .replace(" ", " (?:\[{2}|\]{2}|) ")
                )
                section_search_R = (
                    section + r"(?:(?:[^\{\}]|\n)*?)(</gallery>)"
                ).replace(" ", "(?:\s*|)")
                m = re.search(section_search_R, old_text)
                try:
                    old_section = m.group()
                except AttributeError:
                    section = None

            # First check if we are already on the page,
            # in that case skip. Can happen if the process
            # have been previously interrupted.

            if re.search(wikipattern(file), old_text):
                out(
                    "Skipping addToCategorizedFeaturedList for '%s', page already listed."
                    % self.cleanTitle(),
                    color="lightred",
                )
                return

            # If we found a section, we try to add the image in the section else add to the bottom most gallery (unsorted)
            if section != None:
                file_info = "%s|%s\n</gallery>" % (file, self.cleanTitle())
                updated_section = re.sub(r"</gallery>", file_info, old_section)
                new_text = old_text.replace(old_section, updated_section)
            else:
                # We just need to append to the bottom of the gallery with an added title
                # The regexp uses negative lookahead such that we place the candidate in the
                # last gallery on the page.
                new_text = re.sub(
                    "(?s)</gallery>(?!.*</gallery>)",
                    "%s|%s\n</gallery>" % (file, self.cleanTitle()),
                    old_text,
                    1,
                )

            self.commit(old_text, new_text, page, "Added [[%s]]" % file)

    def getImagePage(self):
        """Get the image page itself."""
        return pywikibot.Page(G_Site, self.fileName())

    def addAssessments(self):
        """
        Adds the the assessments template to a featured
        pictures descripion page.
        This is ==STEP 3== of the parking procedure
        Will add assessments to all files in a set
        """

        if self.isSet():
            files = self.setFiles()
        else:
            files = []
            files.append(self.fileName())
        for file in files:
            page = pywikibot.Page(G_Site, file)
            current_page = page
            old_text = page.get(get_redirect=True)
            AssR = re.compile(r"{{\s*[Aa]ssessments\s*\|(.*)}}")
            fn_or = self.fileName(alternative=False)  # Original filename
            fn_al = self.fileName(alternative=True)  # Alternative filename
            # We add the com-nom parameter if the original filename
            # differs from the alternative filename.
            comnom = (
                "|com-nom=%s" % fn_or.replace("File:", "") if fn_or != fn_al else ""
            )

            # The template needs the com-nom to link to the site from file page
            if self.isSet():
                comnom = "|com-nom=" + (
                    re.search(r"/[Ss]et/(.*)", self.page.title())
                ).group(1)
            else:
                pass

            # First check if there already is an assessments template on the page
            params = re.search(AssR, old_text)
            if params:
                # Make sure to remove any existing com/features or subpage params
                # TODO: 'com' will be obsolete in the future and can then be removed
                # TODO: 'subpage' is the old name of com-nom. Can be removed later.
                params = re.sub(r"\|\s*(?:featured|com)\s*=\s*\d+", "", params.group(1))
                params = re.sub(r"\|\s*(?:subpage|com-nom)\s*=\s*[^{}|]+", "", params)
                params += "|featured=1"
                params += comnom
                if params.find("|") != 0:
                    params = "|" + params
                new_ass = "{{Assessments%s}}" % params
                new_text = re.sub(AssR, new_ass, old_text)
                if new_text == old_text:
                    out(
                        "No change in addAssessments, '%s' already featured."
                        % self.cleanTitle()
                    )
                    return
            else:
                # There is no assessments template so just add it
                if re.search(r"\{\{(?:|\s*)[Ll]ocation", old_text):
                    end = findEndOfTemplate(old_text, "[Ll]ocation")
                elif re.search(r"\{\{(?:|\s*)[Oo]bject[_\s][Ll]ocation", old_text):
                    end = findEndOfTemplate(old_text, "[Oo]bject[_\s][Ll]ocation")
                else:
                    end = findEndOfTemplate(old_text, "[Ii]nformation")

                new_text = (
                    old_text[:end]
                    + "\n{{Assessments|featured=1%s}}\n" % comnom
                    + old_text[end:]
                )
                # new_text = re.sub(r'({{\s*[Ii]nformation)',r'{{Assessments|featured=1}}\n\1',old_text)
                self.commit(old_text, new_text, current_page, "FPC promotion")

    def addToCurrentMonth(self):
        """
        Adds the candidate to the list of featured picture this month

        This is ==STEP 4== of the parking procedure
        """
        if self.isSet():
            files = (self.setFiles())[:1]  # The first file from gallery.
        else:
            files = []
            files.append(self.fileName())
        for file in files:
            FinalVotesR = re.compile(
                r"FPC-results-reviewed\|support=([0-9]{0,3})\|oppose=([0-9]{0,3})\|neutral=([0-9]{0,3})\|"
            )
            NomPagetext = self.page.get(get_redirect=True)
            matches = FinalVotesR.finditer(NomPagetext)
            for m in matches:
                if m is None:
                    ws = wo = wn = "x"
                else:
                    ws = m.group(1)
                    wo = m.group(2)
                    wn = m.group(3)

            today = datetime.date.today()
            monthpage = "Commons:Featured_pictures/chronological/%s %s" % (
                datetime.datetime.utcnow().strftime("%B"),
                today.year,
            )
            page = pywikibot.Page(G_Site, monthpage)
            try:
                old_text = page.get(get_redirect=True)
            except pywikibot.exceptions.NoPageError:
                old_text = ""
            # First check if we are already on the page,
            # in that case skip. Can happen if the process
            # have been previously interrupted.
            if re.search(wikipattern(file), old_text):
                out(
                    "Skipping addToCurrentMonth for '%s', page already listed."
                    % self.cleanTitle(),
                    color="lightred",
                )
                return

            # Find the number of lines in the gallery, if AttributeError set count as 1
            m = re.search(r"(?ms)<gallery>(.*)</gallery>", old_text)
            try:
                count = m.group(0).count("\n")
            except AttributeError:
                count = 1

            # We just need to append to the bottom of the gallery
            # with an added title
            # TODO: We lack a good way to find the creator, so it is left out at the moment

            if count == 1:
                old_text = (
                    "{{subst:FPArchiveChrono}}\n== %s %s ==\n<gallery>\n</gallery>"
                    % (
                        datetime.datetime.utcnow().strftime("%B"),
                        today.year,
                    )
                )
            else:
                pass

            if self.isSet():
                file_title = "'''%s''' - a set of %s files" % (
                    (re.search(r"/[Ss]et/(.*)", self.page.title())).group(1),
                    str(len(self.setFiles())),
                )
            else:
                file_title = self.cleanTitle()

            new_text = re.sub(
                "</gallery>",
                "%s|%d '''%s''' <br> uploaded by %s, nominated by %s,<br> {{s|%s}}, {{o|%s}}, {{n|%s}} \n</gallery>"
                % (
                    file,
                    count,
                    file_title,
                    uploader(file),
                    self.nominator(),
                    ws,
                    wo,
                    wn,
                ),
                old_text,
            )

            self.commit(old_text, new_text, page, "Added [[%s]]" % file)

    def notifyNominator(self):
        """
        Add a template to the nominators talk page

        This is ==STEP 5== of the parking procedure
        """
        talk_link = "User_talk:%s" % self.nominator(link=False)
        talk_page = pywikibot.Page(G_Site, talk_link)

        try:
            old_text = talk_page.get(get_redirect=True)
        except pywikibot.exceptions.NoPageError:
            out(
                "notifyNominator: No such page '%s' but ignoring..." % talk_link,
                color="lightred",
            )
            return

        fn_or = self.fileName(alternative=False)  # Original filename
        fn_al = self.fileName(alternative=True)  # Alternative filename

        # First check if we are already on the page,
        # in that case skip. Can happen if the process
        # have been previously interrupted.

        # We add the subpage parameter if the original filename
        # differs from the alternative filename.
        subpage = "|subpage=%s" % fn_or if fn_or != fn_al else ""

        # notification for set candidates should add a gallery to talk page and
        # it should be special compared to usual promotions.

        if self.isSet():
            if re.search(r"{{FPpromotionSet\|%s}}" % wikipattern(fn_al), old_text):
                return
            files_newline_string = converttostr(self.setFiles(), "\n")
            new_text = old_text + "\n\n== Set Promoted to FP ==\n<gallery mode=packed heights=80px>%s\n</gallery>\n{{FPpromotionSet|%s%s}} /~~~~" % (
                files_newline_string,
                fn_al,
                subpage,
            )
            try:
                self.commit(
                    old_text, new_text, talk_page, "FPC promotion of [[%s]]" % fn_al
                )
            except pywikibot.exceptions.LockedPageError as error:
                out(
                    "Page is locked '%s', but ignoring since it's just the user notification."
                    % error,
                    color="lightyellow",
                )
            return
        else:
            pass

        if re.search(r"{{FPpromotion\|%s}}" % wikipattern(fn_or), old_text):
            out(
                "Skipping notifyNominator for '%s', page already listed at '%s'."
                % (self.cleanTitle(), talk_link),
                color="lightred",
            )
            return

        new_text = old_text + "\n\n== FP Promotion ==\n{{FPpromotion|%s%s}} /~~~~" % (
            fn_al,
            subpage,
        )

        try:
            self.commit(
                old_text, new_text, talk_page, "FPC promotion of [[%s]]" % fn_al
            )
        except pywikibot.exceptions.LockedPageError as error:
            out(
                "Page is locked '%s', but ignoring since it's just the user notification."
                % error,
                color="lightyellow",
            )

    def notifyUploader(self):
        """
        Add a template to the uploaders talk page
        This is ==STEP 6== of the parking procedure
        """
        if self.isSet():
            files = self.setFiles()
        else:
            files = []
            files.append(self.fileName())

        for file in files:
            # Check if nominator and uploaders are same, avoiding adding a template twice
            if self.nominator() == uploader(file, link=True):
                continue

            talk_link = "User_talk:%s" % uploader(file, link=False)
            talk_page = pywikibot.Page(G_Site, talk_link)
            try:
                old_text = talk_page.get(get_redirect=True)
            except pywikibot.exceptions.NoPageError:
                out(
                    "notifyUploader: No such page '%s' but ignoring..." % talk_link,
                    color="lightred",
                )
                return

            fn_or = self.fileName(alternative=False)  # Original filename
            fn_al = self.fileName(alternative=True)  # Alternative filename

            # First check if we are already on the page,
            # in that case skip. Can happen if the process
            # have been previously interrupted.

            if re.search(r"{{FPpromotion\|%s}}" % wikipattern(fn_or), old_text):
                out(
                    "Skipping notifyUploader for '%s', page already listed at '%s'."
                    % (self.cleanTitle(), talk_link),
                    color="lightred",
                )
                return

            # We add the subpage parameter if the original filename
            # differs from the alternative filename.

            subpage = "|subpage=%s" % fn_or if fn_or != fn_al else ""

            if self.isSet():
                subpage = "|subpage=" + (
                    re.search(r"[Ss]et/(.*)", self.page.title())
                ).group(0)
                fn_al = file

            new_text = (
                old_text
                + "\n\n== FP Promotion ==\n{{FPpromotedUploader|%s%s}} /~~~~"
                % (
                    fn_al,
                    subpage,
                )
            )

            try:
                self.commit(
                    old_text, new_text, talk_page, "FPC promotion of [[%s]]" % fn_al
                )
            except pywikibot.exceptions.LockedPageError as error:
                out(
                    "Page is locked '%s', but ignoring since it's just the user notification."
                    % error,
                    color="lightyellow",
                )

    def moveToLog(self, reason=None):
        """
        Remove this candidate from the current list
        and add it to the log of the current month

        This is ==STEP 7== of the parking procedure
        """

        why = (" (%s)" % reason) if reason else ""

        # Add to log
        # (Note FIXME, we must probably create this page if it does not exist)
        today = datetime.date.today()
        current_month = datetime.datetime.utcnow().strftime("%B")
        log_link = "Commons:Featured picture candidates/Log/%s %s" % (
            current_month,
            today.year,
        )
        log_page = pywikibot.Page(G_Site, log_link)

        # If the page does not exist we just create it ( put does that automatically )
        try:
            old_log_text = log_page.get(get_redirect=True)
        except pywikibot.exceptions.NoPageError:
            old_log_text = ""

        if re.search(wikipattern(self.fileName()), old_log_text):
            out(
                "Skipping add in moveToLog for '%s', page already there"
                % self.cleanTitle(),
                color="lightred",
            )
        else:
            new_log_text = old_log_text + "\n{{%s}}" % self.page.title()
            self.commit(
                old_log_text,
                new_log_text,
                log_page,
                "Adding [[%s]]%s" % (self.fileName(), why),
            )

        # Remove from current list
        candidate_page = pywikibot.Page(G_Site, self._listPageName)
        old_cand_text = candidate_page.get(get_redirect=True)
        new_cand_text = re.sub(
            r"{{\s*%s\s*}}.*?\n?" % wikipattern(self.page.title()), "", old_cand_text
        )

        if old_cand_text == new_cand_text:
            out(
                "Skipping remove in moveToLog for '%s', no change." % self.cleanTitle(),
                color="lightred",
            )
        else:
            self.commit(
                old_cand_text,
                new_cand_text,
                candidate_page,
                "Removing [[%s]]%s" % (self.fileName(), why),
            )

    def park(self):
        """
        This will do everything that is needed to park a closed candidate

        1. Check whether the count is verified or not
        2. If verified and featured:
          * Add page to 'Commons:Featured pictures, list'
          * Add to subpage of 'Commons:Featured pictures, list'
          * Add {{Assessments|featured=1}} or just the parameter if the template is already there
            to the picture page (should also handle subpages)
          * Add the picture to the 'Commons:Featured_pictures/chronological/current_month'
          * Add the template {{FPpromotion|File:XXXXX.jpg}} to the Talk Page of the nominator.
        3. If featured or not move it from 'Commons:Featured picture candidates/candidate list'
           to the log, f.ex. 'Commons:Featured picture candidates/Log/August 2009'
        """

        # First make a check that the page actually exist:
        if not self.page.exists():
            out("%s: (no such page?!)" % self.cutTitle())
            return

        # First look for verified results
        text = self.page.get(get_redirect=True)
        results = re.findall(self._VerifiedR, text)

        if not results:
            out("%s: (ignoring, no verified results)" % self.cutTitle())
            return

        if len(results) > 1:
            out("%s: (ignoring, several verified results ?)" % self.cutTitle())
            return

        if self.isWithdrawn():
            out("%s: (ignoring, was withdrawn)" % self.cutTitle())
            return

        if self.isFPX():
            out("%s: (ignoring, was FPXed)" % self.cutTitle())
            return

        # Check if the image page exist, if not we ignore this candidate
        if not pywikibot.Page(G_Site, self.fileName()).exists():
            out("%s: (WARNING: ignoring, can't find image page)" % self.cutTitle())
            return

        # Ok we should now have a candidate with verified results that we can park
        vres = results[0]

        # If the suffix to the title has not been added, add it now
        new_text = self.fixHeader(text, vres[3])
        if new_text != text:
            self.commit(text, new_text, self.page, "Fixed header")

        if vres[3] == "yes":
            self.handlePassedCandidate(vres)
        elif vres[3] == "no":
            # Non Featured picure
            self.moveToLog(self._conString)
        else:
            out(
                "%s: (ignoring, unknown verified feature status '%s')"
                % (self.cutTitle(), vres[3])
            )
            return

    def handlePassedCandidate(self, results):
        """Must be implemented by subclass (do the park procedure for passing candidate)."""
        raise NotImplementedException()

    @staticmethod
    def commit(old_text, new_text, page, comment):
        """
        This will commit new_text to the page
        and unless running in automatic mode it
        will show you the diff and ask you to accept it.

        @param old_text Used to show the diff
        @param new_text Text to be submitted as the new page
        @param page Page to submit the new text to
        @param comment The edit comment
        """

        out("\n About to commit changes to: '%s'" % page.title())

        # Show the diff
        pywikibot.showDiff(
            old_text,
            new_text,
        )

        if G_Dry:
            choice = "n"
        elif G_Auto:
            choice = "y"
        else:
            choice = pywikibot.bot.input_choice(
                "Do you want to accept these changes to '%s' with comment '%s' ?"
                % (page.title(), comment),
                [("yes", "y"), ("no", "n"), ("quit", "q")],
            )

        if choice == "y":
            page.put(new_text, summary=comment, watch=None, minor=False)
        elif choice == "q":
            out("Aborting.")
            sys.exit(0)
        else:
            out("Changes to '%s' ignored" % page.title())

class FPCandidate(Candidate):
    """A candidate up for promotion."""

    def __init__(self, page):
        """Constructor."""
        Candidate.__init__(
            self,
            page,
            SupportR,
            OpposeR,
            NeutralR,
            "featured",
            "not featured",
            ReviewedTemplateR,
            CountedTemplateR,
            VerifiedResultR,
        )
        self._listPageName = "Commons:Featured picture candidates/candidate list"

    def getResultString(self):
        if self.imageCount() > 1:
            return "\n\n{{FPC-results-unreviewed|support=X|oppose=X|neutral=X|featured=no|gallery=|alternative=|sig=<small>'''Note: this candidate has several alternatives, thus if featured the alternative parameter needs to be specified.'''</small> /~~~~)}}"
        else:
            return "\n\n{{FPC-results-unreviewed|support=%d|oppose=%d|neutral=%d|featured=%s|gallery=%s|sig=~~~~}}" % (
                self._pro,
                self._con,
                self._neu,
                "yes" if self.isPassed() else "no",
                self.findGalleryOfFile(),
            )

    def getCloseCommitComment(self):
        if self.imageCount() > 1:
            return "Closing for review - contains alternatives, needs manual count"
        else:
            return (
                "Closing for review (%d support, %d oppose, %d neutral, featured=%s)"
                % (self._pro, self._con, self._neu, "yes" if self.isPassed() else "no")
            )

    def handlePassedCandidate(self, results):

        # Strip away any eventual section
        # as there is not implemented support for it
        fgallery = re.sub(r"#.*", "", results[4])

        # Now addToCategorizedFeaturedList can handle sections within the gallery page
        gallery_without_removing_section = results[4]

        # Check if we have an alternative for a multi image
        if self.imageCount() > 1:
            if len(results) > 5 and len(results[5]):
                if not pywikibot.Page(G_Site, results[5]).exists():
                    out("%s: (ignoring, specified alternative not found)" % results[5])
                else:
                    self._alternative = results[5]
            else:
                out("%s: (ignoring, alternative not set)" % self.cutTitle())
                return

        # Featured picture
        if not len(fgallery):
            out("%s: (ignoring, gallery not defined)" % self.cutTitle())
            return
        self.addToFeaturedList(re.search(r"(.*?)(?:/|$)", fgallery).group(1))
        self.addToCategorizedFeaturedList(gallery_without_removing_section)
        self.addAssessments()
        self.addToCurrentMonth()
        self.notifyNominator()
        self.notifyUploader()
        self.moveToLog(self._proString)

class DelistCandidate(Candidate):
    """A delisting candidate."""

    def __init__(self, page):
        Candidate.__init__(
            self,
            page,
            DelistR,
            KeepR,
            NeutralR,
            "delisted",
            "not delisted",
            DelistReviewedTemplateR,
            DelistCountedTemplateR,
            VerifiedDelistResultR,
        )
        self._listPageName = "Commons:Featured picture candidates/candidate list"

    def getResultString(self):
        return (
            "\n\n{{FPC-delist-results-unreviewed|delist=%d|keep=%d|neutral=%d|delisted=%s|sig=~~~~}}"
            % (self._pro, self._con, self._neu, "yes" if self.isPassed() else "no")
        )

    def getCloseCommitComment(self):
        return "Closing for review (%d delist, %d keep, %d neutral, delisted=%s)" % (
            self._pro,
            self._con,
            self._neu,
            "yes" if self.isPassed() else "no",
        )

    def handlePassedCandidate(self, results):
        # Delistings does not care about the gallery
        self.removeFromFeaturedLists(results)
        self.removeAssessments()
        self.moveToLog(self._proString)

    def removeFromFeaturedLists(self, results):
        """Remove a candidate from all featured lists."""
        # We skip checking the page with the 4 newest images
        # the chance that we are there is very small and even
        # if we are we will soon be rotated away anyway.
        # So just check and remove the candidate from any gallery pages

        references = self.getImagePage().getReferences(withTemplateInclusion=False)
        for ref in references:
            if ref.title().startswith("Commons:Featured pictures/"):
                if ref.title().startswith("Commons:Featured pictures/chronological"):
                    out("Adding delist note to %s" % ref.title())
                    old_text = ref.get(get_redirect=True)
                    now = datetime.datetime.utcnow()
                    new_text = re.sub(
                        r"(([Ff]ile|[Ii]mage):%s.*)\n"
                        % wikipattern(self.cleanTitle(keepExtension=True)),
                        r"\1 '''Delisted %d-%02d-%02d (%s-%s)'''\n"
                        % (now.year, now.month, now.day, results[1], results[0]),
                        old_text,
                    )
                    self.commit(
                        old_text, new_text, ref, "Delisted [[%s]]" % self.fileName()
                    )
                else:
                    old_text = ref.get(get_redirect=True)
                    new_text = re.sub(
                        r"(\[\[)?([Ff]ile|[Ii]mage):%s.*\n"
                        % wikipattern(self.cleanTitle(keepExtension=True)),
                        "",
                        old_text,
                    )
                    self.commit(
                        old_text, new_text, ref, "Removing [[%s]]" % self.fileName()
                    )

    def removeAssessments(self):
        """Remove FP status from an image."""
        imagePage = self.getImagePage()
        old_text = imagePage.get(get_redirect=True)

        # First check for the old {{Featured picture}} template
        new_text = re.sub(
            r"{{[Ff]eatured[ _]picture}}", "{{Delisted picture}}", old_text
        )

        # Then check for the assessments template
        # The replacement string needs to use the octal value for the char '2' to
        # not confuse python as '\12\2' would obviously not work
        new_text = re.sub(
            r"({{[Aa]ssessments\s*\|.*(?:com|featured)\s*=\s*)1(.*?}})",
            r"\1\062\2",
            new_text,
        )

        self.commit(old_text, new_text, imagePage, "Delisted")

def wikipattern(s):
    """Return a string that can be matched against different way of writing it on wikimedia projects."""

    def rep(m):
        if m.group(0) == " " or m.group(0) == "_":
            return "[ _]"
        elif (
            m.group(0) == "("
            or m.group(0) == ")"
            or m.group(0) == "*"
            or m.group(0) == "+"
            or m.group(0) == "="
            or m.group(0) == "?"
            or m.group(0) == "!"
            or m.group(0) == "^"
            or m.group(0) == "-"
        ):
            return "\\" + m.group(0)

    return re.sub(r"[ _()*+=?!^-]", rep, s)

def out(text, newline=True, date=False, color=None):
    """Just output some text to the consoloe or log."""
    if color:
        text = "\03{%s}%s\03{default}" % (color, text)
    dstr = (
        "%s: " % datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
        if date and not G_LogNoTime
        else ""
    )
    pywikibot.stdout("%s%s" % (dstr, text), newline=newline)

def findCandidates(page_url, delist):
    """Finds all candidates on the main FPC page."""
    page = pywikibot.Page(G_Site, page_url)
    candidates = []
    templates = page.templates()
    for template in templates:
        title = template.title()
        if title.startswith(candPrefix):
            # out("Adding '%s' (delist=%s)" % (title,delist))
            if delist and "/removal/" in title:
                candidates.append(DelistCandidate(template))
            elif not delist and "/removal/" not in title:
                candidates.append(FPCandidate(template))
        else:
            pass
            # out("Skipping '%s'" % title)
    return candidates

def checkCandidates(check, page, delist):
    """
    Calls a function on each candidate found on the specified page

    @param check  A function in Candidate to call on each candidate
    @param page   A page containing all candidates
    @param delist Boolean, telling whether this is delistings of fpcs
    """

    if not G_Site.logged_in():
        G_Site.login()

    candidates = findCandidates(page, delist)

    def containsPattern(candidate):
        return candidate.cleanTitle().lower().find(G_MatchPattern.lower()) != -1

    candidates = list(filter(containsPattern, candidates))

    tot = len(candidates)
    i = 1
    for candidate in candidates:

        if not G_Threads:
            out("(%03d/%03d) " % (i, tot), newline=False, date=True)

        try:
            if G_Threads:
                while threading.activeCount() >= config.max_external_links:
                    time.sleep(0.1)
                thread = ThreadCheckCandidate(candidate, check)
                thread.start()
            else:
                check(candidate)
        except pywikibot.exceptions.NoPageError as error:
            out("No such page '%s'" % error, color="lightred")
        except pywikibot.exceptions.LockedPageError as error:
            out("Page is locked '%s'" % error, color="lightred")

        i += 1
        if G_Abort:
            break

def filter_content(text):
    """
    Will filter away content that should not be parsed.

    Currently this includes:
    * The <s> tag for striking out votes
    * The <nowiki> tag which is just for displaying syntax
    * Image notes
    * Html comments

    """
    text = strip_tag(text, "[Ss]")
    text = strip_tag(text, "[Nn]owiki")
    text = strip_tag(text, "[Ss]trike")
    text = strip_tag(text, "[Dd]el")
    text = re.sub(
        r"(?s){{\s*[Ii]mageNote\s*\|.*?}}.*{{\s*[iI]mageNoteEnd.*?}}", "", text
    )
    text = re.sub(r"(?s)<!--.*?-->", "", text)
    return text

def strip_tag(text, tag):
    """Will simply take a tag and remove a specified tag."""
    return re.sub(r"(?s)<%s>.*?</%s>" % (tag, tag), "", text)

def uploader(file, link=True):
    """Return the link to the user that uploaded the nominated image."""
    page = pywikibot.Page(G_Site, file)
    history = page.revisions(reverse=True, total=1)
    for data in history:
        username = data.user
    if not history:
        return "Unknown"
    if link:
        return "[[User:%s|%s]]" % (username, username)
    else:
        return username

def converttostr(input_list, seperator):
    """Make string from list."""
    resultant_string = seperator.join(input_list)
    return resultant_string

def findEndOfTemplate(text, template):
    """
    As regexps can't properly deal with nested parantheses.
    this function will manually scan for where a template ends
    such that we can insert new text after it.
    Will return the position or 0 if not found.
    """
    m = re.search(r"{{\s*%s" % template, text)
    if not m:
        return 0

    lvl = 0
    cp = m.start() + 2

    while cp < len(text):
        ns = text.find("{{", cp)
        ne = text.find("}}", cp)

        # If we see no end tag, we give up
        if ne == -1:
            return 0

        # Handle case when there are no more start tags
        if ns == -1:
            if not lvl:
                return ne + 2
            else:
                lvl -= 1
                cp = ne + 2

        elif not lvl and ne < ns:
            return ne + 2
        elif ne < ns:
            lvl -= 1
            cp = ne + 2
        else:
            lvl += 1
            cp = ns + 2
    # Apparently we never found it
    return 0

# Data and regexps used by the bot

# List of valid templates
# They are taken from the page Commons:Polling_templates and some common redirects
support_templates = (
    "[Ss]upport",
    "[Pp]ro",
    "[Ss]im",
    "[Tt]ak",
    "[Ss]í",
    "[Pp]RO",
    "[Ss]up",
    "[Yy]es",
    "[Oo]ui",
    "[Kk]yllä",  # First support + redirects
    "падтрымліваю",
    "[Pp]our",
    "[Tt]acaíocht",
    "דעב",
    "[Ww]eak support",
    "[Ww]eak [Ss]",
    "[Ss]amþykkt",
    "支持",
    "찬성",
    "[Ss]for",
    "за",
    "[Ss]tödjer",
    "เห็นด้วย",
    "[Dd]estek",
    "[Aa] favore?",
    "[Ss]trong support",
    "[Ss]Support",
    "Υπέρ",
    "[Ww]Support",
    "[Ss]",
    "[Aa]poio",
)
oppose_templates = (
    "[Oo]",
    "[Oo]ppose",
    "[Kk]ontra",
    "[Nn]ão",
    "[Nn]ie",
    "[Mm]autohe",
    "[Oo]pp",
    "[Nn]ein",
    "[Ee]i",  # First oppose + redirect
    "[Cс]упраць",
    "[Ee]n contra",
    "[Cc]ontre",
    "[Ii] gcoinne",
    "[Dd]íliostaigh",
    "[Dd]iscordo",
    "נגד",
    "á móti",
    "反対",
    "除外",
    "반대",
    "[Mm]ot",
    "против",
    "[Ss]tödjer ej",
    "ไม่เห็นด้วย",
    "[Kk]arsi",
    "FPX contested",
    "[Cc]ontra",
    "[Cc]ontrario",
    "[Ss]trong oppose",
    "[Ww]eak oppose",
    "[Ww]eak [Oo]",
)
neutral_templates = (
    "[Nn]eutral?",
    "[Oo]partisk",
    "[Nn]eutre",
    "[Nn]eutro",
    "[Nn]",
    "נמנע",
    "[Nn]øytral",
    "中立",
    "Нэўтральна",
    "[Tt]arafsız",
    "Воздерживаюсь",
    "[Hh]lutlaus",
    "중립",
    "[Nn]eodrach",
    "เป็นกลาง",
    "[Vv]n",
    "[Nn]eutrale",
)
delist_templates = (
    "[Dd]elist",
    "sdf",
)  # Should the remove templates be valid here ? There seem to be no internationalized delist versions
keep_templates = (
    "[Kk]eep",
    "[Vv]k",
    "[Mm]antener",
    "[Gg]arder",
    "維持",
    "[Bb]ehold",
    "[Mm]anter",
    "[Bb]ehåll",
    "เก็บ",
    "保留",
)

#
# Compiled regular expressions follows
#

# Used to remove the prefix and just print the file names
# of the candidate titles.
candPrefix = "Commons:Featured picture candidates/"
PrefixR = re.compile("%s.*?([Ff]ile|[Ii]mage)?:" % candPrefix)

# Looks for result counts, an example of such a line is:
# '''result:''' 3 support, 2 oppose, 0 neutral => not featured.
#
PreviousResultR = re.compile(
    r"'''result:'''\s+(\d+)\s+support,\s+(\d+)\s+oppose,\s+(\d+)\s+neutral\s*=>\s*((?:not )?featured)",
    re.MULTILINE,
)

# Looks for verified results
VerifiedResultR = re.compile(
    r"""
                              {{\s*FPC-results-reviewed\s*\|        # Template start
                              \s*support\s*=\s*(\d+)\s*\|           # Support votes (1)
                              \s*oppose\s*=\s*(\d+)\s*\|            # Oppose Votes  (2)
                              \s*neutral\s*=\s*(\d+)\s*\|           # Neutral votes (3)
                              \s*featured\s*=\s*(\w+)\s*\|          # Featured, should be yes or no, but is not verified at this point (4)
                              \s*gallery\s*=\s*([^|]*)              # A gallery page if the image was featured (5)
                              (?:\|\s*alternative\s*=\s*([^|]*))?   # For candidate with alternatives this specifies the winning image (6)
                              .*}}                                  # END
                              """,
    re.MULTILINE | re.VERBOSE,
)

VerifiedDelistResultR = re.compile(
    r"{{\s*FPC-delist-results-reviewed\s*\|\s*delist\s*=\s*(\d+)\s*\|\s*keep\s*=\s*(\d+)\s*\|\s*neutral\s*=\s*(\d+)\s*\|\s*delisted\s*=\s*(\w+).*?}}",
    re.MULTILINE,
)

# Matches the entire line including newline so they can be stripped away
CountedTemplateR = re.compile(r"^.*{{\s*FPC-results-unreviewed.*}}.*$\n?", re.MULTILINE)
DelistCountedTemplateR = re.compile(
    r"^.*{{\s*FPC-delist-results-unreviewed.*}}.*$\n?", re.MULTILINE
)
ReviewedTemplateR = re.compile(r"^.*{{\s*FPC-results-reviewed.*}}.*$\n?", re.MULTILINE)
DelistReviewedTemplateR = re.compile(
    r"^.*{{\s*FPC-delist-results-reviewed.*}}.*$\n?", re.MULTILINE
)

# Is whitespace allowed at the end ?
SectionR = re.compile(r"^={1,4}.+={1,4}\s*$", re.MULTILINE)
# Voting templates
SupportR = re.compile(
    r"{{\s*(?:%s)(\|.*)?\s*}}" % "|".join(support_templates), re.MULTILINE
)
OpposeR = re.compile(
    r"{{\s*(?:%s)(\|.*)?\s*}}" % "|".join(oppose_templates), re.MULTILINE
)
NeutralR = re.compile(
    r"{{\s*(?:%s)(\|.*)?\s*}}" % "|".join(neutral_templates), re.MULTILINE
)
DelistR = re.compile(
    r"{{\s*(?:%s)(\|.*)?\s*}}" % "|".join(delist_templates), re.MULTILINE
)
KeepR = re.compile(r"{{\s*(?:%s)(\|.*)?\s*}}" % "|".join(keep_templates), re.MULTILINE)
# Finds if a withdraw template is used
# This template has an optional string which we
# must be able to detect after the pipe symbol
WithdrawnR = re.compile(r"{{\s*(?:[wW]ithdrawn?|[fF]PD)\s*(\|.*)?}}", re.MULTILINE)
# Nomination that contain the fpx template
FpxR = re.compile(r"{{\s*FPX(\|.*)?}}", re.MULTILINE)
# Counts the number of displayed images
ImagesR = re.compile(r"\[\[((?:[Ff]ile|[Ii]mage):[^|]+).*?\]\]")
# Look for a size specification of the image link
ImagesSizeR = re.compile(r"\|.*?(\d+)\s*px")
# Find if there is a thumb parameter specified
ImagesThumbR = re.compile(r"\|\s*thumb\b")
# Finds the last image link on a page
LastImageR = re.compile(
    r"(?s)(\[\[(?:[Ff]ile|[Ii]mage):[^\n]*\]\])(?!.*\[\[(?:[Ff]ile|[Ii]mage):)"
)

# Auto reply yes to all questions
G_Auto = False
# Auto answer no
G_Dry = False
# Use threads
G_Threads = False
# Avoid timestamps in output
G_LogNoTime = False
# Pattern to match
G_MatchPattern = ""
# Flag that will be set to True if CTRL-C was pressed
G_Abort = False

def main(*args):
    global G_Auto
    global G_Dry
    global G_Threads
    global G_LogNoTime
    global G_MatchPattern
    global G_Site

    candidates_page = "Commons:Featured picture candidates/candidate_list"
    testLog = "Commons:Featured_picture_candidates/Log/January_2009"

    worked = False
    delist = False
    fpc = False

    # First look for arguments that should be set for all operations
    i = 1
    for arg in sys.argv[1:]:
        if arg == "-auto":
            G_Auto = True
            sys.argv.remove(arg)
            continue
        elif arg == "-dry":
            G_Dry = True
            sys.argv.remove(arg)
            continue
        elif arg == "-threads":
            G_Threads = True
            sys.argv.remove(arg)
            continue
        elif arg == "-delist":
            delist = True
            sys.argv.remove(arg)
            continue
        elif arg == "-fpc":
            fpc = True
            sys.argv.remove(arg)
            continue
        elif arg == "-notime":
            G_LogNoTime = True
            sys.argv.remove(arg)
            continue
        elif arg == "-match":
            if i + 1 < len(sys.argv):
                G_MatchPattern = sys.argv.pop(i + 1)
                sys.argv.remove(arg)
                continue
            else:
                out("Warning - '-match' need a pattern, aborting.", color="lightred")
                sys.exit(0)
        i += 1

    if not delist and not fpc:
        delist = True
        fpc = True

    # Can not use interactive mode with threads
    if G_Threads and (not G_Dry and not G_Auto):
        out("Warning - '-threads' must be run with '-dry' or '-auto'", color="lightred")
        sys.exit(0)

    args = pywikibot.handle_args(*args)
    G_Site = pywikibot.Site()

    # Abort on unknown arguments
    for arg in args:
        if arg not in [
            "-test",
            "-close",
            "-info",
            "-park",
            "-threads",
            "-fpc",
            "-delist",
            "-help",
            "-notime",
            "-match",
            "-auto",
        ]:
            out(
                "Warning - unknown argument '%s' aborting, see -help." % arg,
                color="lightred",
            )
            sys.exit(0)

    for arg in args:
        worked = True
        if arg == "-test":
            if delist:
                out("-test not supported for delisting candidates")
            if fpc:
                checkCandidates(Candidate.compareResultToCount, testLog, delist=False)
        elif arg == "-close":
            if delist:
                out("Closing delist candidates...", color="lightblue")
                checkCandidates(Candidate.closePage, candidates_page, delist=True)
            if fpc:
                out("Closing fpc candidates...", color="lightblue")
                checkCandidates(Candidate.closePage, candidates_page, delist=False)
        elif arg == "-info":
            if delist:
                out("Gathering info about delist candidates...", color="lightblue")
                checkCandidates(Candidate.printAllInfo, candidates_page, delist=True)
            if fpc:
                out("Gathering info about fpc candidates...", color="lightblue")
                checkCandidates(Candidate.printAllInfo, candidates_page, delist=False)
        elif arg == "-park":
            if G_Threads and G_Auto:
                out(
                    "Auto parking using threads is disabled for now...",
                    color="lightyellow",
                )
                sys.exit(0)
            if delist:
                out("Parking delist candidates...", color="lightblue")
                checkCandidates(Candidate.park, candidates_page, delist=True)
            if fpc:
                out("Parking fpc candidates...", color="lightblue")
                checkCandidates(Candidate.park, candidates_page, delist=False)

    if not worked:
        out("Warning - you need to specify an argument, see -help.", color="lightred")

def signal_handler(signal, frame):
    global G_Abort
    print("\n\nReceived SIGINT, will abort...\n")
    G_Abort = True

signal.signal(signal.SIGINT, signal_handler)

if __name__ == "__main__":
    try:
        main()
    finally:
        pywikibot.stopme()


import json
import os

HERE = os.path.split(__file__)[0]

def build_model_search_space(task_type: str, data_path: str, **kwargs):
    search_space = {
        "task_type": {"_type": "choice", "_value": []},
        "data_path": {"_type": "choice", "_value": [data_path]},
    }
    if data_path.endswith(".csv"):
        assert "y_col" in kwargs, "need to specify y_col for csv"
        search_space["y_col"] = {"_type": "choice", "_value": [kwargs["y_col"]]}
        if "X_cols" in kwargs:
            search_space["X_cols"] = {"_type": "choice", "_value": [kwargs["X_cols"]]}
        elif "drop_cols" in kwargs:
            search_space["drop_cols"] = {
                "_type": "choice",
                "_value": [kwargs["drop_cols"]],
            }
        else:
            raise Exception("need to specify either X_cols or drop_cols")

    hps = json.load(open(f"{HERE}/search/model_search/hp.json"))
    models = json.load(open(f"{HERE}/search/model_search/model_select.json"))

    classifier_choices_with_hps = []
    for classifier in models["classification"]:
        classifier_choices_with_hps.append({"_name": classifier, **hps[classifier]})
    regressor_choices_with_hps = []
    for regressor in models["regression"]:
        regressor_choices_with_hps.append({"_name": regressor, **hps[regressor]})

    if task_type == "classification":
        search_space["task_type"]["_value"].append(
            {
                "_name": "classification",
                "model": {"_type": "choice", "_value": classifier_choices_with_hps},
            }
        )
    elif task_type == "regression":
        search_space["task_type"]["_value"].append(
            {
                "_name": "regression",
                "model": {"_type": "choice", "_value": regressor_choices_with_hps},
            }
        )
    else:
        raise Exception(f"unsupported task type {task_type}")
    return search_space

if __name__ == "__main__":
    print(__file__)


from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.template.defaultfilters import slugify
import requests
from bs4 import BeautifulSoup
import pandas as pd

@api_view(['GET'])
def country_info(request, pk=None):
    idk=pk

    country= idk.replace(" ", '_')
    wiki_link = "https://en.wikipedia.org/wiki/" + country

    dic = {
        'flag_link': 'https://upload.wikimedia.org/wikipedia/en/4/41/Flag_of_India.svg',
        'capital': 'New Delhi',
        'largest_city': ['Mumbai', 'New Delhi'],
        'official_languages': ['Hindi', 'English'],
        'area_total': 3287263,
        'Population': '1,352,642,280',
        'GDP_nominal': ' $3.050 trillion',

    }

    table_class="infobox ib-country vcard"
    response=requests.get(wiki_link)
    soup=BeautifulSoup(response.text,'html.parser')
    table=soup.find('table', attrs={'class':table_class})
    rows = table.find_all('tr')
    images = soup.find('img', attrs={'class': "thumbborder"})

    dic['flag_link'] = "https:" +images['src']

    for tr in rows:

        th = tr.find_all('th', attrs={'class': "infobox-label"})
        td = tr.find_all('td', attrs={'class': 'infobox-data'})
        for h in th:

            for d in td:

                if h.text == "Largest city":
                    temp = d.find_all('a')
                    templ = []
                    for t in temp:
                        if t.get("title") != None:
                            templ.append(t.get('title'))
                    dic['largest_city'] = templ
                if h.text == "Capitaland largest city":

                    templ = []
                    temp = d.find_all('a')
                    for t in temp:
                        if t.get('title') != None:
                            templ.append(t.get('title'))
                    dic['capital'] = templ
                    dic['largest_city'] = templ
                    break
                elif h.text == "Capital":
                    temp = d.find_all('a')
                    templ = []

                    for t in temp:
                        if t.get('title') != None:
                            templ.append(t.get('title'))
                    dic['capital'] = templ
                    break

    dic2={}
    dic3={}
    for tr in rows:

        th = tr.find_all('th')
        td = tr.find_all('td')
        for j in th:
            # print(j.text)

            for k in td:
                # print(k.text)
                temp = k.find_all('a')
                temp_list = []
                for t in temp:
                    if t.get('title') != None:

                        temp_list.append(t.get('title'))

                dic3[j.text] = temp_list

            for s in td:
                dic2[j.text] = s.text

    try:
        dic['official_languages'] = dic3['Official languageand national language']
    except:
        dic['official_languages']=dic3['Official\xa0languages']

    try:
        dic['GDP']=dic2['•\xa0Total']
    except:
        dic['GDP']=dic['GDP']
    try:
        dic['Population']=dic2['•\xa02022 estimate']
    except:
        dic['Population']="---------NA---"

    return Response(dic)


import sys

from django.core.management.base import BaseCommand, CommandError
from dynamic.models import Dynamic
from DDHelper import settings
from timeline.tasks import process_timeline
from timeline.models import TimelineDynamicProcessInfo

class Command(BaseCommand):
    help = '重新更新timeline'

    def add_arguments(self, parser):
        pass

    def handle(self, *args, **options):
        all_dynamic = Dynamic.objects\
            .exclude(timelinedynamicprocessinfo__process_version__gte=TimelineDynamicProcessInfo.VERSION)\
            .values_list('dynamic_id', flat=True)
        self.stdout.write(f"共{len(all_dynamic)}条需要更新")
        for i, did in enumerate(all_dynamic):
            process_timeline(did)
            if i % 200 == 0:
                self.stdout.write(f'\r{i}/{len(all_dynamic)}')
                self.stdout.flush()
        self.stdout.write(self.style.SUCCESS("成功调用"))


#The objective of this python script, is to display a review of current python knowledge
#Class:
class Review:#Creates a class in python

	def __init__(self,name, age):#Creates a function with args, initializes classes properties
		self.name = name
		self.age = age
		print("I am "+name+" I am "+age+" years old.")

	def Traveling(self):#Creates a no args function
		place = "Washington"
		destinations = ("California","Washington","Pennsylvania","North Carolina","South Carolina")
		if place in destinations:#Creates a branching statement
			print("I have visited "+place)
		else:
			print("I have not visited "+place)

	def Car(self,year,color,make,model):#Creates a function with args
		print("My car is a "+year+" "+color+" "+make+" "+model)

	def Hobbies(self):
		funStuff = ["Programming","Sitting outside","Listening to music","Going to concerts"]
		for objectives in funStuff:#Creates a looping statement
			print(objectives)

testReview = Review("William","27")
testReview.Traveling()
print("I enjoy: ")
testReview.Hobbies()


from sklearn.model_selection import ParameterGrid
import subprocess
import os
import random

param_dict = {
        # Traing Setting:
        '--model': ['ABWIM_plus', 'HR_BiLSTM_plus'],      #'HR_BiLSTM_plus'/'ABWIM_plus'
        '--dataset': ['wq'],            # dataset_name
        '--framework': ['UHop'],    #'baseline'/'UHop'
        '--train': [' '],
        #'--train1test2': [True],
        '--earlystop_tolerance': [20],

        '--saved_dir':['saved_model'],
#        '--stop_when_err': [''],       # comment to use weighted loss
#        '--step_every_step' : [''],    # comment to step only once in each question
        '--dynamic' : ['recurrent'],         # none/flatten/recurrent
        '--reduce_method' : ['dense'],  # bilstm or dense, for abwim_plus
#        '--only_one_hop' : [''],       # uncomment for one-hop-only training/testing

        # HyperParameter:
        '--epoch_num': [200],
        '--emb_size': [300],
        '--hidden_size': [150, 100],
        '--dropout_rate': [0, 0.2, 0.4],
        '--learning_rate': [0.001, 0.0001, 0.00001],
        '--optimizer': ['rmsprop', 'adadelta'],
        '--neg_sample': [1024],
        '--l2_norm': [0.0],# 0.00000001, 0.000000001],
        '--margin': [0.1, 0.3, 0.5, 0.7, 1.0],
        #'--train_embedding': [True],
        '--hop_weight': [1, 0.8, 1, 1.25],
        '--task_weight': [1],
        '--acc_weight': [1, 0.25, 0.5, 1],
        #'--pretrained_bert': ['bert-base-uncased'],#'bert-base-uncased' or 'bert-base-multilingual-cased'
        '--q_representation': ['lstm']
        }

os.chdir('../src/')
process_str = 'python3 main.py'

################################ NEED TO CONFIGURE ABOVE ####################################################

possible_param_list = list(ParameterGrid(param_dict))
random.shuffle(possible_param_list)
#print(possible_param_list)
print(f'There will be {len(possible_param_list)} runs')
for i, param in enumerate(possible_param_list[:]):
    print(f'{i}/{len(possible_param_list)} run')
    # param is a dict
    run_str = process_str
    for arg, arg_val in param.items():
        run_str += f' {arg} {arg_val}'
    print(run_str)
    process = subprocess.run(run_str.split(), encoding='UTF-8')
    #print(process.stdout.split('\n')[-3])


'''
-Medium-

*Sort*

A group of two or more people wants to meet and minimize the total
travel distance. You are given a 2D grid of values 0 or 1, where each
1 marks the home of someone in the group. The distance is calculated
using Manhattan Distance, where distance(p1, p2) = |p2.x - p1.x| + |p2.y - p1.y|.

Example:

Input:

1 - 0 - 0 - 0 - 1
|   |   |   |   |
0 - 0 - 0 - 0 - 0
|   |   |   |   |
0 - 0 - 1 - 0 - 0

Output: 6

Explanation: Given three people living at (0,0), (0,4), and (2,2):
             The point (0,2) is an ideal meeting point, as the total travel
             distance of 2+2+2=6 is minimal. So return 6.

'''

class Solution(object):

    def shortestDistance(self, grid):
        m, n = len(grid), len(grid[0])
        rows, cols = [], []
        for i in range(m):
            for j in range(n):
                if grid[i][j] == 1:
                    rows.append(i)
                    cols.append(j)
        #rows.sort()
        cols.sort()
        res, i, j = 0, 0, len(rows)-1
        while i < j:
            res += rows[j] - rows[i] + cols[j] - cols[i]
            i += 1
            j -= 1
        return res

    def minTotalDistance(self, grid):
        # Write your code here
        # x,y坐标分别去中位数（如果个数为偶数，则中间2个数之间任取一个）为最佳方案。
        # 时间复杂度 O(MN), O(MLogM), O(N*LogN) 三者取最大值
        # 空间复杂度 O(M+N)
        xLst,yLst = [],[]
        m,n = len(grid),len(grid[0])

        for i in range(m):
            for j in range(n):
                if grid[i][j]:
                    xLst.append(i)
                    yLst.append(j)

        xLst.sort()
        yLst.sort()

        x = xLst[len(xLst)//2]
        y = yLst[len(yLst)//2]

        return sum([abs(x-x1) for x1 in xLst]) + sum([abs(y-y1) for y1 in yLst])

s = Solution()
grid = [[1,0,0,0,1],
        [0,0,0,0,0],
        [0,0,1,0,0]]
print(s.shortestDistance(grid))
print(s.minTotalDistance(grid))


"""Tests for `yaml_params` module."""
import os
import pytest
from collections import OrderedDict
from yaml_params import YAMLParams
from datetime import datetime
import filecmp

def test_init_assert_noargs():
    """Test to make sure YAMLParams asserts when no name argument."""
    with pytest.raises(TypeError):
        myObj = YAMLParams()

def test_init_assert_name_not_string():
    """Test to make sure that the name given is a string."""
    with pytest.raises(TypeError):
        myObj = YAMLParams(42)

def test_init_assert_configdir_not_string():
    """Test to make sure that the config_dir arg is a string."""
    with pytest.raises(TypeError):
        myObj = YAMLParams("my_name", config_dir=42)

def test_init_assert_params_not_dict():
    """Test to make sure that the params arg is a dict."""
    with pytest.raises(TypeError):
        myObj = YAMLParams("my_name", params=42)

def test_init_nameonly_nofile():
    """Test to make sure YAMLParams asserts when no file exists."""
    with pytest.raises(FileNotFoundError):
        myObj = YAMLParams("no_file_with_name")

def test_init_load_file_false():
    """Test to make sure YAMLParams iniailizes correctly w/ load_file=False."""
    myObj = YAMLParams("my_obj", load_file=False)
    dt_str = datetime.now().strftime("%A, %d. %B %Y %I:%M%p")
    assert myObj.params == {}

    assert myObj._params_yaml['info']['name'] == 'my_obj'
    assert myObj._params_yaml['info']['version'] == 'v0.1.0'
    assert myObj._params_yaml['info']['date'] == dt_str
    assert myObj._params_yaml['info']['author'] == 'YAMLParams class, kind: SELF_GENERATED'
    assert myObj._params_yaml['info']['description'] == 'Test parametters for developing YAMLParams class.'
    assert myObj._params_yaml['params'] == OrderedDict()

def test_init_load_file_false_dump():
    """test dump_params_yaml() method for a load_file=False object."""
    myObj = YAMLParams("my_obj", load_file=False)
    dt_str = datetime.now().strftime("%A, %d. %B %Y %I:%M%p")
    assert myObj.dump_params_yaml() == (
        f"# Parameters (params) for YAMLParams object 'my_obj'.\n"
        f"info:\n  name: my_obj\n"
        f"  version: v0.1.0\n"
        f"  date: {dt_str}\n"
        f"  author: 'YAMLParams class, kind: SELF_GENERATED'\n"
        f"  description: Test parametters for developing YAMLParams class.\n"
        f"params: {{}}\n"
    )

def test_init_nameonly_w_file():
    startdir = os.path.abspath(os.path.curdir)
    os.chdir(os.path.join(startdir, 'tests', 'inputs'))
    my_obj = YAMLParams("my_obj")
    os.chdir(startdir)
    assert my_obj._name == "my_obj"
    expected_params = {
                        'myint': 42,
                        'myfloat': 2.718281828,
                        'mystring': 'this is a string',
                        'myintarray': [1,2,3],
                        'mydict': {
                            'myint': 72,
                            'myfloat': 3.1415926,
                            'mystring': 'this is another string',
                            'myfloatarray': [4.0, 5.0, 6.0],
                        }
                       }
    assert my_obj.params == expected_params

def test_init_w_config_dir():
    my_obj = YAMLParams("my_obj", config_dir="tests/inputs")
    assert my_obj._name == "my_obj"
    expected_params = {
                        'myint': 42,
                        'myfloat': 2.718281828,
                        'mystring': 'this is a string',
                        'myintarray': [1,2,3],
                        'mydict': {
                            'myint': 72,
                            'myfloat': 3.1415926,
                            'mystring': 'this is another string',
                            'myfloatarray': [4.0, 5.0, 6.0],
                        }
                       }
    assert my_obj.params == expected_params

def test_dump_loaded_w_comments():
    my_obj = YAMLParams("my_obj", config_dir="tests/inputs")
    assert my_obj.dump_params_yaml() == (
        '# Summary information\n'
        'info:\n  name: "my_name"\n'
        '  version: "v1.0.0"\n'
        '  date: "Saturday, 29. July 2023 02:11AM"\n'
        '  author: "esbailey@me.com"\n'
        '  description: "Test parametters for developing YAMLParams class."\n'
        '\n'
        '# Parameters for use by simualation\n'
        'params:\n'
        '  myint: 42\n'
        '  myfloat: 2.718281828\n'
        '  mystring: "this is a string"  # got a comment here.\n'
        '  myintarray: [1, 2, 3]\n'
        '  mydict:\n'
        '    myint: 72\n'
        '    myfloat: 3.1415926\n'
        '    mystring: "this is another string"\n'
        '    myfloatarray: [4.0, 5.0, 6.0]\n'
    )

def test_init_w_params_dict():
    my_params = {'myfloat':1.234,
                 'mystr':'this is a string.',
                 'mylistoffloats':[1.2,3.4,5.6],
                 'mylistofints':[1,2,3,4,5]
                 }
    my_obj = YAMLParams("my_obj", params=my_params)
    assert my_obj.params == my_params

def test_read_config_file():
    myObj = YAMLParams("my_obj", load_file=False)
    myObj.read_params_config(config_file='tests/inputs/my_obj.yaml')
    assert myObj.dump_params_yaml() == (
        '# Summary information\n'
        'info:\n  name: "my_name"\n'
        '  version: "v1.0.0"\n'
        '  date: "Saturday, 29. July 2023 02:11AM"\n'
        '  author: "esbailey@me.com"\n'
        '  description: "Test parametters for developing YAMLParams class."\n'
        '\n'
        '# Parameters for use by simualation\n'
        'params:\n'
        '  myint: 42\n'
        '  myfloat: 2.718281828\n'
        '  mystring: "this is a string"  # got a comment here.\n'
        '  myintarray: [1, 2, 3]\n'
        '  mydict:\n'
        '    myint: 72\n'
        '    myfloat: 3.1415926\n'
        '    mystring: "this is another string"\n'
        '    myfloatarray: [4.0, 5.0, 6.0]\n'
    )
    expected_params = {
                        'myint': 42,
                        'myfloat': 2.718281828,
                        'mystring': 'this is a string',
                        'myintarray': [1,2,3],
                        'mydict': {
                            'myint': 72,
                            'myfloat': 3.1415926,
                            'mystring': 'this is another string',
                            'myfloatarray': [4.0, 5.0, 6.0],
                        }
                       }
    assert myObj.params == expected_params
    assert myObj._params_yaml_dir.endswith('tests/inputs')
    assert myObj._params_yaml_filepath.endswith('tests/inputs/my_obj.yaml')

def test_save_params_yaml():
    myObj = YAMLParams('my_obj', config_dir='tests/inputs')
    myObj._params_yaml_dir = os.path.abspath(os.path.curdir)
    myObj._params_yaml_filepath = os.path.join(myObj._params_yaml_dir, 'my_obj_saved.yaml')
    myObj.save_params_yaml()
    assert filecmp.cmp('my_obj_saved.yaml',
                       './tests/expected_outputs/my_obj.yaml')
    os.remove('my_obj_saved.yaml')

def test_edit_string_w_comment():
    myObj = YAMLParams('my_obj', config_dir='tests/inputs')
    myObj.params['mystring'] = 'this is a string, modified'
    myObj.save_params_yaml(filepath=os.path.join(os.path.curdir,'my_obj_saved.yaml'))
    assert filecmp.cmp('my_obj_saved.yaml',
                       './tests/expected_outputs/my_obj_mod_mystring.yaml')
    os.remove('my_obj_saved.yaml')

def test_manual_set_params():
    my_obj = YAMLParams('my_obj', load_file=False)
    my_obj.params = {'myint': 1,
                     'myfloat': 1.234,
                     'mybool': False,
                     'mystr': 'this is a string.',
                     'mylistofints': [1,2,3,4,5],
                     'mylistoffloats': [1.23, 4.56, 7.89],
                     'mylistofbools': [False, True, False, False, True],
                     'mylistofstrs': ['one', 'two', 'three and four', 'five']
                     }
    my_obj.capture_params()
    dt_str = datetime.now().strftime("%A, %d. %B %Y %I:%M%p")
    assert my_obj.dump_params_yaml() == (
       f"# Parameters (params) for YAMLParams object 'my_obj'.\n"
       f"info:\n"
       f"  name: my_obj\n"
       f"  version: v0.1.0\n"
       f"  date: {dt_str}\n"
       f"  author: 'YAMLParams class, kind: SELF_GENERATED'\n"
       f"  description: Test parametters for developing YAMLParams class.\n"
       f"params:\n"
       f"  myint: 1\n"
       f"  myfloat: 1.234\n"
       f"  mybool: false\n"
       f"  mystr: this is a string.\n"
       f"  mylistofints: [1, 2, 3, 4, 5]\n"
       f"  mylistoffloats: [1.23, 4.56, 7.89]\n"
       f"  mylistofbools: [false, true, false, false, true]\n"
       f"  mylistofstrs: [one, two, three and four, five]\n"
    )


import logging
import time

from databases import DatabaseHandler
from ec2 import Ec2Handler
from ecs import EcsHandler
from utils import is_substring_in_string

# The only required setup you need to do
looking_for = {
    'TagName': 'Environment',
    'TagValue': 'staging',
    'ECSClusterName': 'staging-main',
    'ECSServices': [
        'svc1-staging',
        'sv2-staging'
    ]
}

####################################
# DO NOT MODIFY BEYOND THIS POINT! #
####################################
# logger setup
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
# AWS resources handlers
rds = DatabaseHandler(looking_for)
ec2 = Ec2Handler(looking_for)
ecs = EcsHandler(looking_for)

def lambda_handler(event, context):
    """
        This is the Lambda handler function. The code will be executed when an event is generated.
    """
    operation_type = event['operation']

    if operation_type == 'start':
        logging.info('Starting everything...')
        rds.start_rds_docd_clusters()
        ec2.start_ec2_instances()
        ecs.start_ecs_services()
        return_msg = f'Successfully started the resources...'
    elif operation_type == 'stop':
        logging.info('Stopping everything...')
        ecs.stop_ecs_services()
        ec2.stop_ec2_instances()
        rds.stop_rds_docd_clusters()
        return_msg = f'Successfully stopped the resources...'
    else:
        return_msg = f'Operation {operation_type} cannot be handled!'

    return return_msg

def full_local_testing():
    """
        Function to mimic calls to the Lambda handler.
        The function will verify the content of each of the 3 possible responses.
        The 600s waiting time is due to the long time RDS & DocDB clusters take to stop.
        This invocation will affect ALL the resources.
    """
    responses = {}

    # Test a stop event
    event = {
        'operation': 'stop'
    }
    responses['stop'] = lambda_handler(event, None)

    logging.warning('Wait 10 mins for everything to stop')
    time.sleep(600)

    # Test a start event
    event = {
        'operation': 'start'
    }
    responses['start'] = lambda_handler(event, None)

    logging.warning('Wait 10 sec for everything to settle')
    time.sleep(10)

    # Test an invalid event
    event = {
        'operation': 'invalid'
    }
    responses['invalid'] = lambda_handler(event, None)

    # Verify the output content
    assert is_substring_in_string('stopped', responses['stop'])
    assert is_substring_in_string('started', responses['start'])
    assert is_substring_in_string('invalid', responses['invalid'])
    logging.info(f'The testing responses are {responses}')

if __name__ == '__main__':
    """
        This code will NOT run in the context of the Lambda function !!!
        It's for local testing purposes only !!!
    """

    # either run the selective testing
    # selective_local_testing(looking_for)

    # or the full lambda code
    full_local_testing()


import collections

class Solution:
    def isIsomorphic(self, s: str, t: str) -> bool:
        mp = {}
        visited = set()
        if len(s) != len(t):
            return False
        for i in range(len(s)):
            if s[i] not in mp:
                if t[i] in visited:
                    return False
                mp[s[i]] = t[i]
                visited.add(t[i])
            else:
                if mp[s[i]] != t[i]:
                    return False
        return True


#_*_ coding:utf-8 _*_
#!/usr/bin/env python

import pg

def operate_postgre_tbl_product():
    try:
        #pgdb_conn = pg.connect(dbname = 'tpc', host = '192.168.103.31', user = 'gpadmin', passwd = '')
        pgdb_conn = pg.connect("host=192.168.103.31 port=5432 dbname=tpc user=gpadmin")

    except Exception as e:
         print(e.args[0])
         return

    sql_desc = "select * from call_center limit 5;"
    for row in pgdb_conn.query(sql_desc).dictresult():
        print(row)

    pgdb_conn.close()

if __name__ == '__main__':
    operate_postgre_tbl_product()


import xarray as xr
import config
import numpy as np
import gsw
import dask

#dask.config.set(scheduler='single-threaded')

class orca(object):
    ''' get model object and process '''

    def __init__(self):
        self.data_path = config.data_path() + 'ORCA/'
        self.ds = {}
        #self.grid_keys = ['_T', '_U', '_V', '_I']
        self.grid_keys = ['_I']
        for pos in self.grid_keys:
            self.ds[pos] = xr.open_mfdataset(self.data_path +
                        '/ORCA_PATCH_*' + pos + '.nc',
                        compat='override',coords='minimal',
                        chunks={'time_counter':10}, decode_cf=True)

    #def calc_mld(self, ref_depth=10,threshold=0.03):
    #    """
    #    Calculated Mixed Layer Depth
    #    Default threshold is 0.03 referenced to 10m
    #    Data should be in format depth,distance
    #    """
    #
    #    mld[i]=(depth[(np.abs((density[:,i]-density[ref_depth,i ]))>=
    #                   threshold)].min())

    #def save_mld(self):
    #    ''' calc mixed layer depth and save '''
    #    mld = xr.DataArray(self.calc_mld(),
    #                       coords={'distance': self.giddy.distance},
    #                       dims='distance')
    #    mld_ds = self.giddy.assign(mld=mld).drop(['salt','temp','dens'])
    #    mld_ds.to_netcdf(self.root + 'Giddy_2020/giddy_mld.nc')

    def save_area_mean_all(self):
        ''' save lateral mean of all data '''

        for grid in self.grid_keys:
            print ('mean :', grid)
            ds = self.ds[grid].mean(['X','Y']).load()
            for key in ds.keys():
                ds = ds.rename({key: key + '_mean'})
            ds.to_netcdf(self.data_path + 'ORCA_PATCH_mean'
                        + grid + '.nc')

    def save_area_std_all(self):
        ''' save lateral standard deviation of all data '''

        for grid in self.grid_keys:
            print ('std :', grid)
            ds = self.ds[grid].std(['X','Y']).load()
            for key in ds.keys():
                ds = ds.rename({key: key + '_std'})
            ds.to_netcdf(self.data_path + 'ORCA_PATCH_std'
                         + grid + '.nc')

m = orca()
m.save_area_mean_all()
m.save_area_std_all()


import sys
import re
from bs4 import BeautifulSoup, Tag, NavigableString

with open(sys.argv[1]) as fp:
    soup = BeautifulSoup(fp, 'lxml-xml')

root = soup.find_all(value='ROOT')[0]

def find_leaf(tag, phrase):
    tmp_wd = []

    if(tag['value'] == 'CC'):
        cc_wd = tag.find_all('leaf')[0]['value']

        prev_sib = list(tag.previous_siblings)[1]
        prev_sib_depth = str(len(list(prev_sib.parents)))
        prev_sib1 = prev_sib['value']+prev_sib_depth

        next_sib = list(tag.next_siblings)[1]
        next_sib_depth = str(len(list(next_sib.parents)))
        next_sib1 = next_sib['value'] + next_sib_depth

        phrase.append([tag['value']+str(len(list(tag.parents))), prev_sib1, next_sib1, cc_wd])

    #elif (tag.name != 'leaf'):
    elif (re.match( r'^NP|WHNP|PP|WHPP|ADJP|WHADJP|ADVP|WHAVP|X|SBAR|NAC|NML|CONJP|FRAG|INTJ|LST|NAC|NX|QP|PRC|PRN|PRT|QP|RRC|UCP|ROOT|S|,$', tag['value']) and (tag.name != 'leaf')):
        for wd in tag.find_all('leaf'):
                tmp_wd.append(wd['value'])
        phrase.append([tag['value']+str(len(list(tag.parents))), tmp_wd])

def find_lwg(tag,tmp_lwg,flag):
    for tag1 in tag.contents:
        if (isinstance(tag1, Tag)):
            if (re.match(r'^VB|RB|TO|MD', tag1['value'])):
                tmp_lwg.append(tag1.find_all('leaf')[0]['value'])

            elif (re.match(r'^VP', tag1['value'])):
                find_lwg(tag1,tmp_lwg,1)
    if(flag == 0):
        phrase.append([tag['value']+'_LWG'+str(len(list(tag.parents))), tmp_lwg])

def process_tag(tag, phrase):
        if (isinstance(tag, Tag)):
            if (tag['value'] == 'VP') and (tag.parent['value'] != 'VP'):
                tmp_lwg = []
                #find_leaf(tag)
                find_lwg(tag,tmp_lwg,0)
                for tag1 in tag.contents:
                    if (isinstance(tag1, Tag)):
                        process_tag(tag1, phrase)

            elif(re.match(r'CC',tag['value'])):
                find_leaf(tag, phrase)

            else:
                find_leaf(tag, phrase)
                for tag1 in tag.contents:
                    if (isinstance(tag1, Tag)):
                        process_tag(tag1, phrase)

phrase = []
for ch in root.contents:
    if(isinstance(ch, Tag)):
        process_tag(ch,phrase)

for p in phrase:
    if(re.match(r'CC',p[0])):
        print(p[0], ':', p[1],'-', p[3], '-', p[2])
    else:
        print(p[0],':',' '.join(p[1]))


# get_ipython().magic('matplotlib inline')
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
import functools
from scipy.stats import poisson

# Generate training data for sales probability regression
def generate_train_data(B=100):

    def rank(a, p):
        return np.argsort(np.hstack((a, p)))[:,0]

    our_price = 10 + np.random.uniform(0, 10, (B, 1))
    competitor_prices = 10 + np.random.uniform(0, 10, (B, 5))
    our_rank = np.reshape(rank(our_price, competitor_prices), (B, 1))
    X = np.hstack((our_price, competitor_prices, our_rank))

    # Y = (our_rank == 0).astype(int).ravel()
    # Y = np.maximum(0, (3 - our_rank)).astype(int).ravel()
    Y = np.round(np.random.uniform(0, 1, our_rank.shape) * (1 - our_rank / 11)).ravel()

    return (X, Y)

X, Y = generate_train_data()

# Train sales probability model
def make_model(X, Y):
    regr = linear_model.LinearRegression()
    regr.fit(X, Y)
    def predict(x):
        return np.maximum(0, regr.predict(x))
        # return regr.predict_proba(x)[:, 1]
    return predict

sales_model = make_model(X, Y)

T=20
N=5
price_range=np.arange(10, 20, 0.1)
L=0.01
delta=0.99
Z=1

# competitor_prices = 10 + np.random.uniform(0, 10, 5)
competitor_prices = np.array([14, 15, 16, 17, 18])
computed_values = {}

def rank(a, p):
    _rank = p.shape[0]
    for i in range(p.shape[0]):
        if a < p[i]:
            _rank = _rank - 1
    return _rank

def _V(price, t, n):
    x = np.hstack((price, competitor_prices, rank(price, competitor_prices))).reshape(1, -1)
    # sales_prob = round(sales_model(x)[0])
    sales_prob = sales_model(x)[0]

    _sum = 0
    # TODO: Check here
    # for i in range(2):

    # print(sales_prob)

    pi_sum = 0
    for i in range(int(poisson.ppf(0.9999, sales_prob)) + 1):
        pi = poisson.pmf(i, sales_prob)
        pi_sum += pi
        today_profit = min(n, i) * price
        holding_costs = n * L
        _, V_future = V(t + 1, max(0, n - i))
        exp_future_profits = delta * V_future
        _sum += pi * (today_profit - holding_costs + exp_future_profits)
    # print(pi_sum)
    return _sum

def V(t, n):
    if (t,n) in computed_values:
        return computed_values[t,n]
    if t >= T:
        computed_values[t,n] = (0, n * Z)
        return (0, n * Z)
    if n <= 0:
        computed_values[t,n] = (0, 0)
        return (0, 0)

    V_opt = -100000
    price_opt = -100000
    for _, price in enumerate(price_range):
        v = _V(price, t, n)
        if v > V_opt:
            V_opt = v
            price_opt = price

    computed_values[t,n] = (price_opt, V_opt)
    return (price_opt, V_opt)

V(0, N)

print(computed_values)

for i_n in range(N + 1):
    datapoints = [ computed_values[i_t, i_n][0] for i_t in range(T + 1) if(i_t, i_n) in computed_values ]
    print(datapoints)
    label_text = 'N=' + str(i_n)
    plt.plot(datapoints, label=label_text)
    plt.ylabel('n')
    plt.xlabel('T')
    plt.legend()
    plt.show()


"""Contains class for a player in a card game.

Classes:

    Player

Typical usage examples:

    player = Player(1, "John")
"""

from model.card_game.card_group import CardGroup

class Player:
    """Class to represent a player in a card game.

    Attributes:
        id: An int to uniquely identify a player.
        name: A string describing this player.
        hand: A CardGroup for the Cards this player is holding.
    """

    def __init__(self, id: int, name: str):
        """Initializes instance.

        Args:
            id: An unique integer id.
            name: A string for the player's name.
        """
        self.id = id
        self.name = name
        self.hand = CardGroup("%s's hand" % (name))


from imports import *

accu_score = []
f1_score = []
precision =[]
mse = []
mae =[]
recall = []

from sklearn.model_selection import KFold
k = 10
cv = KFold(n_splits=k)

def build(df, x_np, y_np, x_names, y_names):

    for train_index, test_index in cv.split(df):
        x_np_train, y_np_train = x_np[train_index], y_np[train_index]
        x_np_test, y_np_test = x_np[test_index], y_np[test_index]

        scaler = StandardScaler()
        x_np_train= scaler.fit_transform(x_np_train)
        x_np_test = scaler.transform(x_np_test)

        model = RandomForestClassifier(
                                            n_estimators=100,
                                           max_features="sqrt",max_depth=4,max_samples= None,
                                           criterion="gini",
                                           min_impurity_decrease=0,
                                           # # min_samples_leaf= 1, ##only used for H2o comparison
                                           min_samples_split=1,  ##only used for Pyspark comparison
                                           random_state=42)

        ##TRAINING THE MODEL
        model.fit(x_np_train, y_np_train)
        preds = model.predict(x_np_test)

        ##FINDING METRICS
        accur = accuracy_score(y_np_test, preds)
        precise = precision_score(y_np_test, preds)
        MAE = mean_absolute_error(y_np_test, preds)
        MSE = mean_squared_error(y_np_test, preds)
        rcall = recall_score(y_np_test, preds)
        F1 = 2 * (precise * rcall) / (precise + rcall)

        accu_score.append(accur)
        precision.append(precise)
        mse.append(MSE)
        mae.append(MAE)
        recall.append(rcall)
        f1_score.append(F1)

    print(" ")
    print(f"Accuracy : {np.mean(accu_score)}")
    print(f"F1_Score : {np.mean(f1_score)}")
    print(f"Precision: {np.mean(precision)}")
    print(f"M A E    : {np.mean(mae)}")
    print(f"M S E    :{np.mean(mse)}")
    print(f"Recall   : {np.mean(recall)}")
    print(" ")