import os
import platform
import sys
from methods import get_compiler_version, using_gcc
from platform_methods import detect_arch

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from SCons import Environment

def get_name():
    return "LinuxBSD"

def can_build():
    if os.name != "posix" or sys.platform == "darwin":
        return False

    pkgconf_error = os.system("pkg-config --version > /dev/null")
    if pkgconf_error:
        print("Error: pkg-config not found. Aborting.")
        return False

    return True

def get_opts():
    from SCons.Variables import BoolVariable, EnumVariable

    return [
        EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")),
        BoolVariable("use_llvm", "Use the LLVM compiler", False),
        BoolVariable("use_x11", "Use X11 display server", True),
        BoolVariable("use_wayland", "Use Wayland display server", False),
        BoolVariable("use_gles", "Use OpenGL ES 3.2 instead of OpenGL 3.2", False),
    ]

def get_flags():
    return [
        ("arch", detect_arch()),
    ]

def configure(env: "Environment"):
    # Validate arch.
    supported_arches = ["x86_32", "x86_64", "arm32", "arm64"]
    if env["arch"] not in supported_arches:
        print(
            'Unsupported CPU architecture "%s" for Linux / *BSD. Supported architectures are: %s.'
            % (env["arch"], ", ".join(supported_arches))
        )
        sys.exit(255)

    if env["debug_symbols"]:
        env.Append(LINKFLAGS=["-rdynamic"])

    host_is_64_bit = sys.maxsize > 2**32
    if host_is_64_bit and env["arch"] == "x86_32":
        env.Append(CCFLAGS=["-m32"])
        env.Append(LINKFLAGS=["-m32"])
    elif not host_is_64_bit and env["arch"] == "x86_64":
        env.Append(CCFLAGS=["-m64"])
        env.Append(LINKFLAGS=["-m64"])

    ## Compiler configuration

    if "CC" in env and "clang" in os.path.basename(env["CC"]):
        # Convenience check to enforce the use_llvm overrides when CC is clang(++)
        env["use_llvm"] = True

    if env["use_llvm"]:
        if "clang" not in os.path.basename(env["CC"]):
            env["CC"] = "clang"
        env.extra_suffix = ".llvm" + env.extra_suffix

    if env["linker"] != "default":
        print("Using linker program: " + env["linker"])
        env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]])

    # LTO

    if env["lto"] == "auto":  # Full LTO for production.
        env["lto"] = "full"

    if env["lto"] != "none":
        if env["lto"] == "thin":
            if not env["use_llvm"]:
                print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
                sys.exit(255)
            env.Append(CCFLAGS=["-flto=thin"])
            env.Append(LINKFLAGS=["-flto=thin"])
        elif not env["use_llvm"] and env.GetOption("num_jobs") > 1:
            env.Append(CCFLAGS=["-flto"])
            env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))])
        else:
            env.Append(CCFLAGS=["-flto"])
            env.Append(LINKFLAGS=["-flto"])

        if not env["use_llvm"]:
            env["RANLIB"] = "gcc-ranlib"
            env["AR"] = "gcc-ar"

    env.Append(CCFLAGS=["-pipe"])

    ## Dependencies
    ## TODO: Need Deps

    env.Append(
        CPPDEFINES=[
            "LINUXBSD_ENABLED",
            "UNIX_ENABLED",
        ]
    )

    env.Append(LIBS=["pthread"])

    if platform.system() == "Linux":
        env.Append(LIBS=["dl"])

    if platform.system() == "FreeBSD":
        env.Append(LINKFLAGS=["-lkvm"])

    if env["use_wayland"]:
        env.Append(CPPDEFINES=["WAYLAND_ENABLED"])

        if os.system("pkg-config --exists wayland-client"):
            print("Error: Wayland libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-client --cflags --libs")

        if os.system("pkg-config --exists wayland-cursor"):
            print("Error: Wayland cursor libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-cursor --cflags --libs")

        if os.system("pkg-config --exists wayland-egl"):
            print("Error: Wayland EGL libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-egl --cflags --libs")

        if os.system("pkg-config --exists wayland-protocols"):
            print("Error: Wayland protocols not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config wayland-protocols --cflags --libs")

        wayland_protocols_dir = os.popen("pkg-config wayland-protocols --variable=pkgdatadir").read().strip().replace("//", "/")
        print("Wayland protocols dir: " + wayland_protocols_dir)
        os.system("wayland-scanner client-header %s/stable/xdg-shell/xdg-shell.xml %s/xdg-shell.gen.h" % (wayland_protocols_dir, "platform/linuxbsd/wayland"))
        os.system("wayland-scanner private-code %s/stable/xdg-shell/xdg-shell.xml %s/xdg-shell.gen.c" % (wayland_protocols_dir, "platform/linuxbsd/wayland"))

    if env["use_x11"]:
        env.Append(CPPDEFINES=["X11_ENABLED"])

        if os.system("pkg-config --exists x11"):
            print("Error: X11 libraries not found. Aborting.")
            sys.exit(255)
        env.ParseConfig("pkg-config x11 --cflags --libs")


import argparse
import multiprocessing
import subprocess
import sys
from datetime import datetime

mutex = multiprocessing.Lock()

def build(wheel):
    print("{}: building {}".format(datetime.now(), wheel), file=sys.stderr)
    process = subprocess.run(
        [sys.executable, "-m", "build", "--wheel", "--no-isolation"], cwd=wheel
    )
    if process.returncode != 0:
        # If the build without isolation fails, try to build it again with
        # isolation. We need to protect this following logic in two ways:
        #   1. Only build one at a time in this process.
        #   2. Retry a few times to get around flakiness.
        success = False
        for _ in range(0, 3):
            with mutex:
                process = subprocess.run(
                    [sys.executable, "-m", "build", "--wheel"],
                    encoding="utf-8",
                    cwd=wheel,
                    capture_output=True,
                )
                if process.returncode != 0:
                    continue
                success = True
                break
        if not success:
            print("STDOUT:", file=sys.stderr)
            print(process.stdout, file=sys.stderr)
            print("STDERR:", file=sys.stderr)
            print(process.stderr, file=sys.stderr)
            raise RuntimeError(
                "{}: ERROR: failed to build {}".format(datetime.now(), wheel)
            )

def main(jobs, wheels):
    with multiprocessing.Pool(jobs) as pool:
        results = []
        for wheel in wheels:
            result = pool.apply_async(build, args=(wheel,))
            results.append(result)
        pool.close()
        for result in results:
            result.get()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Builds Python wheels.")
    parser.add_argument(
        "wheels",
        metavar="wheel",
        type=str,
        nargs="+",
        help="A path to the extracted wheel directory.",
    )
    parser.add_argument(
        "--jobs",
        type=int,
        default=8,
        help="The number of concurrent build jobs to be executed.",
    )
    args = parser.parse_args()
    exit(main(args.jobs, args.wheels))


from django.conf import settings
from django.contrib.sites.shortcuts import get_current_site
from django.core.mail import send_mail
from django.core.paginator import Paginator
from django.db import models
from django.template import loader
from django.utils import timezone

from modelcluster.fields import ParentalKey
from modelcluster.contrib.taggit import ClusterTaggableManager
from taggit.models import Tag, TaggedItemBase

from wagtail.admin.panels import (
    FieldPanel,
    MultiFieldPanel,
)
from wagtail.contrib.routable_page.models import RoutablePageMixin, route
from wagtail import blocks
from wagtail.blocks import ListBlock
from wagtail.blocks.list_block import ListValue
from wagtail.fields import RichTextField, StreamField
from wagtail.models import Page
from wagtail.models.collections import Collection
from wagtail.rich_text import RichText
from wagtail.embeds.blocks import EmbedBlock
from wagtail.images import get_image_model
from wagtail.images.models import Image, AbstractImage, AbstractRendition
from wagtail.search import index
from wagtail.snippets.models import register_snippet

from wagtailmedia.models import Media, AbstractMedia

from dateutil import tz
from django_comments.moderation import CommentModerator
from django_comments_xtd.moderation import moderator
from honeypot.decorators import honeypot_equals

from .blocks import ImageBlock, MediaBlock, SlideBlock, SlidesBlock
from .utils import get_elided_page_range

import datetime as dt
import pyexiv2
import re

# ···························································
# CUSTOM BASE PAGE MODEL AND MISC PAGE(S)
# ···························································

class BasePage(Page):
    def get_context(self, request):
        context = super().get_context(request)
        try:
            context['theme_class'] = request.COOKIES['themeClass']
        except KeyError:
            context['theme_class'] = 'theme-light'
        return context

    class Meta:
        abstract = True

class ComplexPage(BasePage):
    body = StreamField([
        ('heading', blocks.CharBlock(classname="full title")),
        ('paragraph', blocks.RichTextBlock(
            features=[
                'h3',
                'h4',
                'h5',
                'ol',
                'ul',
                'blockquote',
                'bold',
                'italic',
                'superscript',
                'subscript',
                'strikethrough',
                'code',
                'link',
                'image',
                'document-link',
            ],
        )),
        ('image', ImageBlock()),
        ('slides', SlidesBlock()),
        ('embed', EmbedBlock()),
        ('media', MediaBlock(icon='media')),
        ('raw_HTML', blocks.RawHTMLBlock(required=False)),
        ('danger', blocks.RawHTMLBlock(label='DANGER!', required=False)),
    ], use_json_field=True)
    page_message = RichTextField()

    content_panels = Page.content_panels + [
        FieldPanel('page_message'),
        FieldPanel('body'),
    ]

class BasicPage(BasePage):
    body = RichTextField(
        blank=True,
        features=[
            'h3',
            'h4',
            'h5',
            'ol',
            'ul',
            'blockquote',
            'bold',
            'italic',
            'superscript',
            'subscript',
            'strikethrough',
            'code',
            'link',
            'image',
            'document-link',
        ],
    )

    content_panels = Page.content_panels + [
        FieldPanel('body', classname='full'),
    ]

class GalleryPage(BasePage):
    body = StreamField([
        ('heading', blocks.CharBlock(classname="full title")),
        ('paragraph', blocks.RichTextBlock(
            features=[
                'blockquote',
                'bold',
                'italic',
                'superscript',
                'subscript',
                'strikethrough',
                'link',
                'image',
                'document-link',
            ],
        )),
        ('slides', SlidesBlock()),
        ('embed', EmbedBlock()),
        ('media', MediaBlock(icon='media')),
        ('raw_HTML', blocks.RawHTMLBlock(required=False)),
    ], use_json_field=True)

    # fields for the create-from-collection functionality
    import_collection = models.ForeignKey(
        Collection,
        blank=True,
        on_delete=models.SET_NULL,
        null=True,
    )
    common_alt_text = models.CharField(
        blank=True,
        help_text='This alt text will be applied to each new slide.',
        max_length=256,
    )
    common_caption = RichTextField(
        blank=True,
        help_text='This caption will be applied to each new slide.',
        features=['bold', 'italic', 'link'],
    )

    content_panels = Page.content_panels + [
        MultiFieldPanel(
            [
                FieldPanel('import_collection'),
                FieldPanel('common_alt_text'),
                FieldPanel('common_caption'),
            ],
            heading='Create Slides from a Collection',
            classname='collapsible collapsed',
            help_text='Slides will be created in this page\'s Body when you Save.',
        ),
        FieldPanel('body'),
    ]

    settings_panels = Page.settings_panels + [
        FieldPanel('first_published_at'),
    ]

    def get_exif_description(self, image):
        """
        Return an image's EXIF ImageDescription or None.
        """
        f = image.file.file.open()
        i = pyexiv2.ImageData(f.read())
        exif_desc = i.read_exif().get('Exif.Image.ImageDescription')
        i.close()
        return exif_desc

    @property
    def page_message(self):
        return self.get_parent().specific.page_message

    def save_revision(self, **kwargs):
        """
        If an import_collection is provided, append its images to the body in a
        SlidesBlock. Include common_alt_text and common_caption in each slide
        if provided. Clear these three fields before moving on.
        """
        if self.import_collection:

            i = get_image_model()
            images = i.objects.filter(collection=self.import_collection).order_by('file')

            slides_block = []
            for image in images:
                # check for EXIF ImageDescription, prefer over common_caption
                exif_desc = self.get_exif_description(image)
                caption = exif_desc or self.common_caption
                slide_block = {
                    'image': image,
                    'caption': RichText(caption),
                    'alt_text': self.common_alt_text,
                }
                slides_block.append(slide_block)
            # as of Wagtail 4.1, we must cast explicitly to a ListValue
            # containing a ListBlock of SlideBlocks -- YUCK! :(
            slides_block = ListValue(ListBlock(SlideBlock()), values=slides_block)
            self.body.append(('slides', {'slides': slides_block}))

            self.import_collection = None
            self.common_alt_text = ''
            self.common_caption = ''

        return super(GalleryPage, self).save_revision(**kwargs)

# ···························································
# SUPPORTING CLASSES: TAGS AND SIDEBAR ELEMENTS
# ···························································

class PostTag(TaggedItemBase):
    content_object = ParentalKey(
        Page,
        on_delete=models.CASCADE,
        related_name='tagged_items',
    )

@register_snippet
class TagDescription(models.Model):
    tag = models.OneToOneField(
        Tag,
        on_delete=models.CASCADE,
    )
    description = RichTextField(
        features=[
            'bold',
            'italic',
            'superscript',
            'subscript',
            'strikethrough',
            'code',
            'link',
        ],
    )

    panels = [
        FieldPanel('tag'),
        FieldPanel('description'),
    ]

    def __str__(self):
        return self.tag.name

    class Meta:
        ordering = ['tag__name']

class BlogSidebar():
    def archives_by_tag(self):
        top = ['micro', 'snapshot']
        post_tags = PostTag.tags_for(Page).order_by('name')
        bottom_tags = []
        top_tags = []
        for tag in post_tags:
            if tag.name in top:
                top_tags.append(tag)
            else:
                bottom_tags.append(tag)
        content = '
'
        for tag in top_tags:
            content += self.get_tag_link_li(tag)
        content += '

'
        for tag in bottom_tags:
            content += self.get_tag_link_li(tag)
        content += '
'
        return content

    def archives_by_year(self):
        idx = self.get_idx_obj()
        years = [d.year for d in Page.objects.dates('first_published_at', 'year')]
        years.reverse()
        content = '
'
        for year in years:
            start_date = dt.datetime(year, 1, 1)
            end_date = dt.datetime(year + 1, 1, 1)
            count = (
                Page.objects.descendant_of(idx)
                .filter(first_published_at__gte=start_date)
                .filter(first_published_at__lt=end_date)
                .count()
            )
            if count:
                href = idx.url + idx.reverse_subpage('posts_by_date', kwargs={'year': year})
                # ideally this would be in a template somewhere
                content += f'
{year} ({count})
'
        content += '
'
        return content

    def blogroll(self):
        page = BasicPage.objects.get(title='Blogroll')
        return page.body

    def get_tag_link_li(self, tag):
        count = PostTag.objects.filter(tag=tag).count()
        idx = self.get_idx_obj()
        href = idx.url + idx.reverse_subpage('posts_by_tag', kwargs={'tag': tag})
        # ideally this would be in a template somewhere
        content = f'
#{tag} ({count})
'
        return content

    def get_idx_obj(self):
        # NOTE! This code assumes there will only ever be one BlogIndex page.
        # (there must be a better way to do this...)
        return BlogIndex.objects.get()

# ···························································
# BLOG POSTS: LEGACY (FROM DRUPAL) AND MODERN (WAGTAIL)
# ···························································

class BlogPost(models.Model, BlogSidebar):
    has_comments_enabled = models.BooleanField(
        default=True,
        verbose_name='Comments enabled',
    )
    tags = ClusterTaggableManager(through=PostTag, blank=True)

    def get_absolute_url(self):
        """
        Returns an absolute URL for the page. (Required by django_comments_xtd.)
        """
        return self.full_url

    def page_message(self):
        return self.get_parent().specific.page_message

    @property
    def post_date(self):
        if self.first_published_at:
            return self.first_published_at
        elif self.latest_revision:
            return self.latest_revision.created_at
        else:
            return dt.datetime.now()

    class Meta:
        abstract = True

class LegacyPost(BasePage, BlogPost):
    body = models.TextField()
    old_path = models.CharField(max_length=64)

    content_panels = Page.content_panels + [
        FieldPanel('old_path'),
        FieldPanel('body', classname='full'),
        FieldPanel('tags'),
        FieldPanel('has_comments_enabled'),
    ]

    settings_panels = Page.settings_panels + [
        FieldPanel('first_published_at'),
    ]

    search_fields = Page.search_fields + [
        index.SearchField('title'),
        index.SearchField('body'),
    ]

    subpage_types = []

class ModernPost(BasePage, BlogPost):
    body = StreamField([
        ('heading', blocks.CharBlock(classname='full title')),
        ('teaser', blocks.RichTextBlock(
            features=[
                'ol',
                'ul',
                'blockquote',
                'bold',
                'italic',
                'superscript',
                'subscript',
                'strikethrough',
                'code',
                'link',
                'image',
                'document-link',
            ],
        )),
        ('paragraph', blocks.RichTextBlock(
            features=[
                'ol',
                'ul',
                'blockquote',
                'bold',
                'italic',
                'superscript',
                'subscript',
                'strikethrough',
                'code',
                'link',
                'image',
                'document-link',
            ],
        )),
        ('image', ImageBlock()),
        ('slides', SlidesBlock()),
        ('embed', EmbedBlock()),
        ('media', MediaBlock(icon='media')),
        ('raw_HTML', blocks.RawHTMLBlock(required=False)),
    ], use_json_field=True)

    do_not_tweet = models.BooleanField(
        default=False,
        verbose_name='Do not tweet',
        help_text='Do not automatically generate a tweet about this post.',
    )
    tweet_title = models.TextField(
        blank=True,
        max_length=256,
    )

    content_panels = Page.content_panels + [
        FieldPanel('body'),
        FieldPanel('tags'),
        FieldPanel('has_comments_enabled'),
    ]

    promote_panels = [
        FieldPanel('do_not_tweet'),
        FieldPanel('tweet_title'),
    ] + Page.promote_panels

    search_fields = Page.search_fields + [
        index.SearchField('title'),
        index.SearchField('body'),
    ]

    subpage_types = []

    def full_clean(self, *args, **kwargs):
        """ Override method to provide default slug for micro posts. """
        if self.is_micro() and not self.url:
            base_slug = dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%MZ')
            self.slug = self._get_autogenerated_slug(base_slug)
        super(ModernPost, self).full_clean(*args, **kwargs)

    def is_micro(self):
        return True if 'micro' in self.tags.names() else False

    def is_snapshot(self):
        return True if 'snapshot' in self.tags.names() else False

    @property
    def micropost_title(self):
        """
        Return a generic title for a micropost.
        These are used in mnmnwag/feeds.py and
        mnmnwag/management/commands/send_tweets.py.
        """
        if self.is_micro():
            home_zone = tz.gettz(settings.TIME_ZONE)
            try:
                post_date = self.first_published_at.astimezone(home_zone).strftime('%Y-%m-%d %I:%M %p')
            except AttributeError:
                # post is not yet published
                post_date = self.latest_revision.created_at.astimezone(home_zone).strftime('%Y-%m-%d %I:%M %p')
            return f'micropost ({post_date})'
        else:
            return self.title

# ···························································
# THE BLOG INDEX
# ···························································

class BlogIndex(RoutablePageMixin, BasePage, BlogSidebar):
    page_message = RichTextField()
    max_count = 1
    subpage_types = ['ModernPost']

    content_panels = Page.content_panels + [
        FieldPanel('page_message'),
    ]

    def get_posts(self):
        return Page.objects.descendant_of(self).live().order_by('-first_published_at')

    def get_modern_posts(self):
        return ModernPost.objects.descendant_of(self).live().order_by('-first_published_at')

    def paginate_posts(self, request, posts):
        paginator = Paginator(posts, 20)
        page_number = request.GET.get('page')
        display_range = get_elided_page_range(
            num_pages=paginator.num_pages,
            page_range=paginator.page_range,
            number=page_number,
        )
        paginator.display_range = display_range
        return paginator.get_page(page_number)

    @route(r'^$')
    def posts_main(self, request):
        """
        This is the "main" blog index route, which lives at /blog.
        Only show ModernPosts here so that the pager is not dominated by
        ancient stuff. LegacyPosts are available in the other routes.
        """
        self.posts = self.paginate_posts(request, self.get_modern_posts())
        return self.serve(request)

    @route(r'^(?P<year>20\d\d)\/?(?P<month>[0-1][0-9])?\/?(?P<day>[0-3][0-9])?/$')
    def posts_by_date(self, request, year, month='', day='', *args, **kwargs):
        if day:
            start_date = dt.datetime(int(year), int(month), int(day))
            display_day = int(day)
            display_month = start_date.strftime("%B")
            end_date = start_date + dt.timedelta(days=1)
            self.index_title = f'Posts from {display_day} {display_month} {year}'
        elif month:
            start_date = dt.datetime(int(year), int(month), 1)
            start_month = start_date.month
            display_month = start_date.strftime("%B")
            start_year = start_date.year
            end_month = 1 if start_date.month == 12 else start_month + 1
            end_year = start_year + 1 if start_date.month == 12 else start_year
            end_date = dt.datetime(end_year, end_month, 1)
            self.index_title = f'Posts from {display_month} {start_year}'
        else:
            start_date = dt.datetime(int(year), 1, 1)
            end_date = dt.datetime(int(year) + 1, 1, 1)
            self.index_title = f'Posts from {year}'
        self.posts = self.paginate_posts(
            request,
            self.get_posts().filter(first_published_at__gte=start_date).filter(first_published_at__lt=end_date),
        )
        return self.serve(request, *args, **kwargs)

    @route(r'^tag/(?P<tag>[-\w]+)/$')
    def posts_by_tag(self, request, tag, *args, **kwargs):
        post_tags = PostTag.objects.filter(tag__slug=tag.lower())
        post_ids = [item.content_object_id for item in post_tags]
        self.posts = self.paginate_posts(
            request,
            self.get_posts().filter(id__in=post_ids),
        )
        self.index_title = f'Posts tagged #{tag}'
        try:
            self.tag_description = TagDescription.objects.get(tag__slug=tag.lower()).description
        except TagDescription.DoesNotExist:
            pass
        return self.serve(request, *args, **kwargs)

# ···························································
# COMMENT MODERATOR CLASS & REGISTRATION
# ···························································

class BlogPostModerator(CommentModerator):
    email_notification = True
    enable_field = 'has_comments_enabled'

    def allow(self, comment, content_object, request):
        """
        Override parent method to include django-honeypot in the mix,
        and to add a few custom anti-spam rules of our own.
        """
        # here to END_OF_DIFFS = our additions to this method

        if not self.verify_honeypot(request):
            return False

        # disallow any comment with Cyrillic characters in it
        # (Russian spam-bots are the absolute worst!)
        if bool(re.search('[а-яА-Я]', comment.comment)):
            return False

        # disallow common pattern as of Feb. 2022:
        #   email ending in .ru with a link in the comment
        # (Russian spam-bots are the absolute worst!)
        if (
            bool(re.search(r'\.ru$', comment.user_email))
            and 'http' in comment.comment
        ):
            return False

        # disallow common pattern as of Mar. 2022:
        #   multiple .ru links in comment body
        # (Russian spam-bots are the absolute worst!)
        if len(re.findall(r'https?:\/\/.*?\.ru', comment.comment)) > 2:
            return False

        # END_OF_DIFFS

        if self.enable_field:
            if not getattr(content_object, self.enable_field):
                return False
        if self.auto_close_field and self.close_after is not None:
            close_after_date = getattr(content_object, self.auto_close_field)
            if close_after_date is not None and self._get_delta(timezone.now(),
                                                                close_after_date).days >= self.close_after:
                return False
        return True

    def email(self, comment, content_object, request):
        """
        Override parent method to inject request into notification template
        context.
        """
        if not self.email_notification:
            return
        recipient_list = [manager_tuple[1] for manager_tuple in settings.MANAGERS]
        t = loader.get_template('comments/comment_notification_email.txt')
        c = {
            'comment': comment,
            'content_object': content_object,
            'request': request,  # this line is our only alteration to this method
        }
        subject = ('[%(site)s] New comment posted on "%(object)s"') % {
            'site': get_current_site(request).name,
            'object': content_object,
        }
        message = t.render(c)
        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, recipient_list, fail_silently=True)

    def moderate(self, comment, content_object, request):
        """
        Override parent method. Moderate all messages unless superuser is
        posting.
        """
        if request.user.is_superuser:
            return False
        else:
            return True

    def verify_honeypot(self, request, field_name=None):
        """
        Modded version of honeypot.validators.verify_honeypot_value().
        Returns True or False; to be used in allow() in this class.
        """
        verifier = getattr(settings, "HONEYPOT_VERIFIER", honeypot_equals)
        if request.method == "POST":
            field = field_name or settings.HONEYPOT_FIELD_NAME
            if field not in request.POST or not verifier(request.POST[field]):
                return False
        return True

moderator.register(LegacyPost, BlogPostModerator)
moderator.register(ModernPost, BlogPostModerator)

# ···························································
# REJECTED COMMENTS   (rejected by Akismet; display in admin)
# ···························································

class RejectedComment(models.Model):
    related_page = models.ForeignKey(Page, on_delete=models.CASCADE)
    submit_date = models.DateTimeField()
    name = models.CharField(max_length=50)
    email = models.EmailField()
    url = models.URLField(blank=True)
    comment = models.TextField(max_length=3000)
    ip_address = models.GenericIPAddressField(blank=True, null=True)

# ···························································
# CUSTOM MEDIA CLASSES                       (for future use)
# ···························································

class CustomImage(AbstractImage):
    # Add any extra fields to image here

    # eg. To add a caption field:
    # caption = models.CharField(max_length=255, blank=True)

    admin_form_fields = Image.admin_form_fields + (
        # Then add the field names here to make them appear in the form:
        # 'caption',
    )

class CustomRendition(AbstractRendition):
    image = models.ForeignKey(CustomImage, on_delete=models.CASCADE, related_name='renditions')

    class Meta:
        unique_together = (
            ('image', 'filter_spec', 'focal_point_key'),
        )

class CustomMedia(AbstractMedia):
    # Add any extra fields to image here

    # eg. To add a caption field:
    # caption = models.CharField(max_length=255, blank=True)

    admin_form_fields = Media.admin_form_fields + (
        # Then add the field names here to make them appear in the form:
        # 'caption',
    )

# ···························································
# SINGLETON ABSTRACT CLASS & ITS CHILDREN
# ···························································

class SingletonModel(models.Model):
    """
    Defines a model with only one row. Taken from:
    https://steelkiwi.com/blog/practical-application-singleton-design-pattern/
    """
    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        self.pk = 1
        super(SingletonModel, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        pass

    @classmethod
    def load(cls):
        obj, created = cls.objects.get_or_create(pk=1)
        return obj

class TweetsTracker(SingletonModel):
    last_run_at = models.DateTimeField(auto_now=True)

    class Meta:
        verbose_name_plural = 'Tweets job tracker'


# encdoing: utf-8
"""
@Project: pm25_v1
@File:    plotmap2
@Author:  Jiachen Zhao
@Time:    2021/8/2 14:30
@Description:
"""

import os
# noinspection PyUnresolvedReferences
import numpy as np
import pandas as pd
import folium
import datetime
from IPython.display import HTML
from src.utils.utils import *
from folium.plugins import HeatMap

root_path = os.path.abspath('.')
data_path = rf'{root_path}/Datasets'
# data_path = rf'F:/data-list/PycharmProjects/Datasets/'
# data_path = r'/data1/zhaojiachen/Datasets/PM25/'
data_path_2021 = data_path + 'pm25-2021/'
data_path_2020 = data_path + 'pm25-2020/'
preprocess_data_path = data_path + 'pm25-preprocess_data/'
preprocess_EachHour_2021 = preprocess_data_path + '21-EachHourEachFile'
preprocess_EachStation_2021 = preprocess_data_path + '21-EachStationEachFile'

preprocess_EachHour_2020 = preprocess_data_path + '20-EachHourEachFile'
preprocess_EachStation_2020 = preprocess_data_path + '20-EachStationEachFile'
preprocess_EachStation_NotFilled_2020 = preprocess_EachStation_2020 + '/NotFilled'
preprocess_EachStation_Filled_2020 = preprocess_EachStation_2020 + '/Filled'

project_folder = "F:/data-list/PycharmProjects/pm25_v1"
lookup_folder = project_folder + "/lookup"
stationMap_folder = project_folder + "/StationMap"
neighborMap_folder = project_folder + "/StationNeighborMap"

def plot_baseMap():
    ch_map = folium.Map(location=[35, 100], zoom_start=4)
    # ch_map = folium.Map(
    #     location=[38.96, 117.78],
    #     zoom_start=4,
    #     # tiles='http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}', # 高德街道图
    #     # tiles='http://webst02.is.autonavi.com/appmaptile?style=6&x={x}&y={y}&z={z}', # 高德卫星图
    #     tiles='https://mt.google.com/vt/lyrs=s&x={x}&y={y}&z={z}',  # google 卫星图
    #     # tiles='https://mt.google.com/vt/lyrs=h&x={x}&y={y}&z={z}', # google 地图
    #     attr='default'
    # )

    folium.Choropleth(
        geo_data='china.json',
        name='China regions',
        fill_opacity=0.1,
        line_opacity=0.2,
    ).add_to(ch_map)
    return ch_map

def plot_stationIcon_fun(query="华北地区", map=None):  # 将输入地区省份的所有站点，在地图上标注出来
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()  # csv中的所有省份
    print(provinces)

    if query in provinces:
        query = [query]
    elif query == "全国":
        query = provinces
    elif query == "华北地区":
        query = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        query = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        query = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        query = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        query = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        query = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        query = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    print(query)
    stations_query_df = stations_df[stations_df['province'].isin(query)]  # 过滤出所选地区的省份数据
    print(stations_query_df.shape)

    folium.LayerControl().add_to(map)
    for idx, row in stations_query_df.iterrows():  # 编号，行信息
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))
        pop = folium.Popup(html=folium.Html("""
                            {}

                            {}

                            {}

                            """.format(s1, s2, s3),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map)
    return map

def provinces_fun():  # 获取csv中所有的省份名，返回省份名列表
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = []
    for ii in stations_df['province'].unique().tolist():
        if ii is not np.NAN:
            if '\u4e00' <= ii[0] <= '\u9fff':
                provinces.append(ii)
    return provinces

def plot_stationMaps():  # 画出所有的站点数据点
    queries = [
        "全国",
        "华北地区", "华东地区", "东北地区", "华中地区", "华南地区", "西南地区", "西北地区",
        # "北京市", '天津市', '河北省', '山西省', '内蒙古自治区',
        # '上海市', '江苏省', '浙江省', '山东省', '安徽省',
        # '辽宁省', '吉林省', '黑龙江省',
        # '湖北省', '湖南省', '河南省', '江西省',
        # '广东省', '广西壮族自治区', '海南省', '福建省',
        # '四川省', '重庆市', '贵州省', '云南省', '西藏自治区',
        # '陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区'
    ]
    for query in queries:
        map_base = plot_baseMap()  # 多张图，所有区域都画出
        plot_stationIcon_fun(query)
        # map_query = plot_stationIcon_fun(query, map_base)
        # map_query.save(f'{stationMap_folder}/{query}.html')

def add_StationByDF(map, df):  # 将传入的 dataframe 中的所有站点数据点显示在Map上
    for idx, row in df.iterrows():
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))
        pop = folium.Popup(html=folium.Html("""
                            {}

                            {}

                            {}

                            """.format(s1, s2, s3),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map)
    return map

def plot_OneStationNeighbor_fun(query_id=None):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    station_net_df = pd.read_csv(lookup_folder + '/StationNeighbors.csv', index_col=0)
    # print(station_net_df.head())
    # print(stations_df.head())
    map_base = plot_baseMap()
    folium.LayerControl().add_to(map_base)
    query_station = stations_df[stations_df['id'] == query_id]  # 选出输入序号站点的数据
    map = add_StationByDF(map_base, query_station)  # 将输入序号站点加入地图
    for i in range(1, 10):  # 若干相邻点
        neighbor_id_i = station_net_df[station_net_df['id'] == query_id][f'{i}_neighbor'].values[0]  # 得到目标站点对应的附近站点id值
        neighbor_dist_i = station_net_df[station_net_df['id'] == query_id][f'{i}_distance'].values[0]
        # print(neighbor_id_i, neighbor_dist_i)
        neighbor_station_i = stations_df[stations_df['id'] == neighbor_id_i]  # 得到附近站点的相关信息
        # print(neighbor_station_i)
        # print([query_station['latitude'].values[0], query_station['longitude'].values[0]])
        # print([neighbor_station_i['latitude'].values[0], neighbor_station_i['longitude'].values[0]])
        folium.PolyLine([[query_station['latitude'].values[0], query_station['longitude'].values[0]],
                         [neighbor_station_i['latitude'].values[0], neighbor_station_i['longitude'].values[0]]],
                        color='red').add_to(map)
        map = add_StationByDF(map, neighbor_station_i)  # 给出对应附近站点的标记
    return map

def plot_DotMap(query=None, Time=None):  # 某个时间下，某个地区范围内所有站点的空气质量等级颜色图（某站点一定半径范围内）
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()

    if query in provinces:
        querys = [query]
    elif query == "全国":
        querys = provinces
    elif query == "华北地区":
        querys = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        querys = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        querys = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        querys = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        querys = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        querys = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        querys = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    stations_query_df = stations_df[stations_df['province'].isin(querys)]

    map_base = plot_baseMap()
    # folium.LayerControl().add_to(map_base)
    aqi_map_feature = folium.map.FeatureGroup()

    for idx, row in stations_query_df.iterrows():
        id = row['id']
        try:
            file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
            df_station_observation = pd.read_csv(file, index_col=0)
            # print(df_station_observation.head())
            df_station_time = df_station_observation[df_station_observation['time'] == Time]  # 获取某个时间点下站点的数据
            pm25_value = df_station_time['pm25'].values[0]
            aqi, rank, quality, alarm_color = compute_iaqi(cp=pm25_value, return_rank=True)  # 空气质量指数aqi，等级，质量评价，警报等级
            # print(f"AQI: {np.round(aqi, 2)}, rank: {rank}, quality: {quality}, alarm_color: {alarm_color}")
            # print(pm25_value)
            # print([row['latitude'], row['longitude']])
            aqi_map_feature.add_child(  # 站点附近一定范围内，显示空气质量aqi对应颜色
                folium.CircleMarker(
                    [row['latitude'], row['longitude']],
                    radius=7,
                    color=None,
                    fill=True,
                    fill_color=alarm_color,
                    fillopacity=10
                )
            )
        except FileNotFoundError:
            print(f'There is no fill {preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv')
            continue
    map_base.add_child(aqi_map_feature)
    map_base.save(f'{project_folder}/AQI_Dot_map/{query}_dotmap_{Time}.html')

def plot_HeatMap(query='全国', Time=20020520):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    provinces = provinces_fun()

    if query in provinces:
        querys = [query]
    elif query == "全国":
        querys = provinces
    elif query == "华北地区":
        querys = ('北京市', '天津市', '河北省', '山西省', '内蒙古自治区')
    elif query == "华东地区":
        querys = ('上海市', '江苏省', '浙江省', '山东省', '安徽省')
    elif query == "东北地区":
        querys = ('辽宁省', '吉林省', '黑龙江省')
    elif query == "华中地区":
        querys = ('湖北省', '湖南省', '河南省', '江西省')
    elif query == "华南地区":
        querys = ('广东省', '广西壮族自治区', '海南省', '福建省')
    elif query == "西南地区":
        querys = ('四川省', '重庆市', '贵州省', '云南省', '西藏自治区')
    elif query == '西北地区':
        querys = ('陕西省', '甘肃省', '新疆维吾尔自治区', '青海省', '宁夏回族自治区')
    else:
        print(f"{query}, 错误的省份")
    stations_query_df = stations_df[stations_df['province'].isin(querys)]

    map_base = plot_baseMap()

    df_haetmap = pd.DataFrame({'id': [], 'latitude': [], 'longitude': [], 'aqi': []})
    for idx, row in stations_query_df.iterrows():
        id = row['id']
        try:
            file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
            df_station_observation = pd.read_csv(file, index_col=0)
            df_station_time = df_station_observation[df_station_observation['time'] == Time]  # 所选区域中某站点id，在所选时间下的站点数据
            pm25_value = df_station_time['pm25'].values[0]  # 取PM2.5浓度数据
            aqi, rank, quality, alarm_color = compute_iaqi(cp=pm25_value, return_rank=True)
            df_haetmap = df_haetmap.append({
                                            'id': id,
                                            'latitude': row['latitude'],
                                            'longitude': row['longitude'],
                                            'aqi': aqi
                                            }, ignore_index=True)  # 重建 df ，专为热力图，是输入 Time 下的某id站点的空气质量
        except FileNotFoundError:
            print(f'There is no fill {preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv')
            continue
    data = df_haetmap[['latitude', 'longitude', 'aqi']].values.tolist()  # 从 df 转为 array,最后转为 list
    map_base.add_child(HeatMap(data, radius=10, gradient={0.16: 'green', 0.32: 'yellow', 0.48: 'orange',
                                                         0.49: 'red', 0.8: 'purple', 1: 'maroon'}))
    print(df_haetmap)
    map_base.save(f'{project_folder}/AQI_heat_map/{query}_heatmap_{Time}.html')

def read_one_observation(id=None, Time=None, columns=['pm25', 'pm10']):  # 某站点id，单个时间点下，某些特征
    file = f'{preprocess_EachStation_Filled_2020}/Filled_{id}_2020.csv'
    df_station_observation = pd.read_csv(file, index_col=0)
    dfrow_station_time = df_station_observation[df_station_observation['time'] == Time]
    df_observation = dfrow_station_time[columns]
    return df_observation

def read_multiple_time(id=None, start_t=None, hours=6, direction='backward', columns=['time','pm25', 'pm10']):
    # a = '/'.join(['20'+str(start_t)[i:i+2] if i==0 else str(start_t)[i:i+2] for i in range(0, len(str(start_t)), 2)])
    begin = datetime.datetime.strptime(f'20{start_t}', '%Y%m%d%H')
    if direction == 'forward':
        delta = datetime.timedelta(hours=1)  # 时间差往前一小时
        tt = [(begin + delta * i).strftime('%Y%m%d%H')[2:] for i in range(hours)]  # 得到一个时间列表 [20010100, 20010101, ……]之类的
    elif direction == 'backward':
        delta = datetime.timedelta(hours=-1)
        tt = [(begin + delta * i).strftime('%Y%m%d%H')[2:] for i in range(hours)][::-1]  # 列表取反，时间仍然转为正序，与 'forward' 保持一致：时间点靠前的序号小
    tt = list(map(int, tt))  # 将字符串转换为int型
    dfs = pd.concat([read_one_observation(id=id, Time=t, columns=columns) for t in tt])
    dfs.reset_index(inplace=True, drop=True)  # 重置索引
    print(dfs)  # 时间前靠前的数据，索引小

def plot_OneStation_pm25_pop(id=None, Time=None):
    stations_df = pd.read_csv(lookup_folder + '/StationCity.csv', index_col=0)
    station_info = stations_df[stations_df['id']==id]
    for idx, row in station_info.iterrows():
        observation = read_one_observation(id=id, Time=Time, columns=['pm25'])
        aqi, rank, quality, alarm_color = compute_iaqi(cp=observation.values, return_rank=True)
        print(f"AQI: {np.round(aqi, 2)}, rank: {rank}, quality: {quality}, alarm_color: {alarm_color}")
        map_base = plot_baseMap()
        folium.LayerControl().add_to(map_base)
        s1 = f"ID: {int(row['id'])}"
        s2 = f"{row['latitude'], row['longitude']}"
        s3 = f"{row['province'], row['city'], row['district']}"
        s4 = f"AQI:      {np.round(aqi, 2)}"
        s5 = f"RANK:     {rank}"
        s6 = f"QUALITY:  {quality}"
        s7 = f"ALARM:    {alarm_color}"
        WIDTH = max(len(s1.encode('utf-8')), len(s2.encode('utf-8')), len(s3.encode('utf-8')))

        pop = folium.Popup(html=folium.Html("""
                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    {}

                                    """.format(s1, s2, s3, '-'*len(s2), s4, s5, s6, s7),
                                            script=True,
                                            width=WIDTH * 4),
                           parse_html=True,
                           max_width=3000)
        folium.Marker(location=[row['latitude'], row['longitude']],
                      popup=pop).add_to(map_base)

        map_base.save(f'{project_folder}/AQI_pop_map/{id}_{Time}.html')

if __name__ == "__main__":
    print('-----')
    # plot_stationMaps()
    # query_plotOneStationNeighbor = [54499, 54511, 58367, 58035]
    # for query_id in query_plotOneStationNeighbor:
    #     map_OneStationNeighbor = plot_OneStationNeighbor_fun(query_id)
    #     map_OneStationNeighbor.save(f'{neighborMap_folder}/{query_id}_neighbors.html')
    # plot_DotMap(query='全国', Time=20020520)
    # read_one_observation(id = 54499, Time=20010508, columns=['pm25', 'pm10'])
    # read_multiple_time(id=54499, start_t=20010501, hours=6)
    # plot_OneStation_pm25_pop(id=58035, Time=20070107)
    # plot_OneStation_pm25_pop(id=54499, Time=20121109)
    # plot_HeatMap(query='河北省', Time=20112520)


import pygame

pygame.init()

SCREEN_WIDTH = 1500
SCREEN_HEIGHT = 800

screen = pygame.display.set_mode((SCREEN_WIDTH,SCREEN_HEIGHT))
pygame.display.set_caption("Solar System")

#---Framerate---#
clock = pygame.time.Clock()
FPS = 60

#----Define Colours---#
BG = (144, 201, 120)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)
PINK = (235, 65, 54)
YELLOW = (236, 232, 11)

#---Define Fonts---#
MainFont = pygame.font.SysFont("Futura", 30)

#---Define Param---#
G = 6.6741*(10**(-11))
ratio = 20 / 149597870700 #20px = 1AU
oldratio = ratio
dt = 30000

#Sun body
xS = SCREEN_WIDTH / 2
yS = SCREEN_HEIGHT / 2
MS = 1.989*(10**30)

#---Load Images / Make Rects---#
sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (10, 10))
sun_rect = sun_image.get_rect()
sun_rect.center = (xS, yS)

#---Classes---#
class PlanetCircle(pygame.sprite.Sprite):
    def __init__(self, vx, vy, x, y, M, R, displayR, colour):
        pygame.sprite.Sprite.__init__(self)

        self.vx = vx
        self.vy = vy
        self.x = x
        self.y = y
        self.M = M
        self.R = R
        self.displayR = displayR
        self.colour = colour

        self.rect = pygame.draw.circle(screen, colour, (xS + x * ratio, yS + y * ratio), displayR)

    def update(self):
        self.x += self.vx * dt
        self.y += self.vy * dt
        self.vx -= dt * (G * MS * self.x) / (((self.x**2 + self.y**2)**0.5)**3)
        self.vy -= dt * (G * MS * self.y) / (((self.x**2 + self.y**2)**0.5)**3)

        self.rect.center = (xS + self.x * ratio, yS + self.y * ratio)

    def draw(self):
        pygame.draw.circle(screen, self.colour, self.rect.center, self.displayR)

class PlanetImage(pygame.sprite.Sprite):
    def __init__(self, vpe, pe, ap, a, b, M, R, planet, displayR):
        pygame.sprite.Sprite.__init__(self)

        self.vx = 0
        self.vy = vpe
        self.x = pe
        self.y = 0
        self.vpe = vpe
        self.pe = pe
        self.ap = ap
        self.a = a
        self.b = b
        self.M = M
        self.R = R
        self.displayR = displayR
        self.planet = planet
        self.changevar = False

        img = pygame.image.load(f"img\\Planets\\{planet}").convert_alpha() #f string for easy path
        self.image = pygame.transform.scale(img, (displayR * 2, displayR * 2))

        self.rect = self.image.get_rect()
        self.rect.center = (xS + self.x * ratio, yS + self.y * ratio)

    def update(self):
        self.x += self.vx * dt #ensure in this order as others are unstable
        self.y += self.vy * dt
        self.vx -= dt * (G * MS * self.x) / (((self.x**2 + self.y**2)**0.5)**3)
        self.vy -= dt * (G * MS * self.y) / (((self.x**2 + self.y**2)**0.5)**3)

        if self.changevar == True:
            self.change()
            self.changevar = False

        self.rect.center = (xS + self.x * ratio, yS + self.y * ratio)

        self.draw()

    def draw(self):
        if self.pe * ratio + xS < SCREEN_WIDTH and self.pe * ratio + xS > 0: #still messes up when moving left/right/up/down AND zooming in/out at the same time, bug on pygame.draw.ellipse part?
            #draw trail
            pygame.draw.ellipse(screen, WHITE, (xS - self.ap * ratio, yS - self.b * ratio, 2 * self.a * ratio, 2 * self.b * ratio), 1) #display, colour, (left, top, width, height)
        #draw planet
        screen.blit(self.image, self.rect)

    def change(self):
        img = pygame.image.load(f"img\\Planets\\{self.planet}").convert_alpha() #f string for easy path
        self.image = pygame.transform.scale(img, (int(self.displayR * 2 * ratio/oldratio), int(self.displayR * 2 * ratio/oldratio)))

        self.rect = self.image.get_rect()

mercury = PlanetImage(61826, 4.3*(10**10), 7.0*(10**10), 5.65*(10**10), 5.49*(10**10), 3.285*(10**23), 2439700, "mercury.png", 1) #vpe, pe, ap, a, b, M, R, planet, displayR
venus = PlanetImage(35252, 1.07*(10**11), 1.09*(10**11), 1.08*(10**11), 1.08*(10**11), 4.867*(10**24), 6051800, "venus.png", 3)
earth = PlanetImage(30282, 1.47*(10**11), 1.52*(10**11), 1.5*(10**11), 1.5*(10**11), 5.972*(10**24), 6371000, "earth.png", 4)
mars = PlanetImage(26489, 2.07*(10**11), 2.49*(10**11), 2.28*(10**11), 2.27*(10**11), 6.39*(10**23), 3389500, "mars.png", 2)
jupiter = PlanetImage(13708, 7.41*(10**11), 8.17*(10**11), 7.79*(10**11), 7.78*(10**11), 1.898*(10**27), 69911000, "jupiter.png", 20)
saturn = PlanetImage(10180, 1.35*(10**12), 1.51*(10**12), 1.43*(10**12), 1.43*(10**12), 5.683*(10**26), 58232000, "saturn.png", 15)
uranus = PlanetImage(7115, 2.74*(10**12), 3.01*(10**12), 2.88*(10**12), 2.87*(10**12), 8.681*(10**25), 25362000, "uranus.png", 12)
neptune = PlanetImage(5478, 4.46*(10**12), 4.54*(10**12), 4.50*(10**12), 4.50*(10**12), 1.024*(10**26), 24622000, "neptune.png", 10)

#---Create Sprite Groups---#
planet_group = pygame.sprite.Group()

#-Planet Group-#
planet_group.add(mercury, venus, earth, mars, jupiter, saturn, uranus, neptune)

#---Main Game Loop---#
run = True
while run: #while run is true

    clock.tick(FPS)

    for event in pygame.event.get():
        #---Quit Game---#
        if event.type == pygame.QUIT:
            run = False
        #---Keys---#
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                run = False

            if event.key == pygame.K_z: #zoom in
                for planet in planet_group:
                    planet.changevar = True

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio)))
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)

                ratio += 10 / 149597870700
            if event.key == pygame.K_x: #zoom out
                for planet in planet_group:
                    planet.changevar = True #change planet scale

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio))) #change sun scale
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)

                ratio -= 10 / 149597870700

            if event.key == pygame.K_i: #go up
                yS += 20
                for planet in planet_group:
                    planet.changevar = True

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio)))
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)
            if event.key == pygame.K_k: #go down
                yS -= 20
                for planet in planet_group:
                    planet.changevar = True

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio)))
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)
            if event.key == pygame.K_j: #go left
                xS += 20
                for planet in planet_group:
                    planet.changevar = True

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio)))
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)
            if event.key == pygame.K_l: #go right
                xS -= 20
                for planet in planet_group:
                    planet.changevar = True

                sun_image = pygame.transform.scale(pygame.image.load(f"img\\sun.png").convert_alpha(), (int(10 * ratio/oldratio), int(10 * ratio/oldratio)))
                sun_rect = sun_image.get_rect()
                sun_rect.center = (xS, yS)

    screen.fill(BLACK)

    screen.blit(sun_image, sun_rect)

    planet_group.update()

    pygame.display.update()

pygame.quit()


from pathlib import Path
import json
import yaml
import requests
import sys

g_config = {"server": {}, "loaded": False}

def load_config():
    if not g_config["loaded"]:
        config_file = str(Path.home()) + "/.ghpcfe/config"
        p = Path(config_file)
        if p.is_file():
            with p.open("r", encoding="utf-8") as f:
                g_config.update(yaml.safe_load(f)["config"])
                g_config["loaded"] = True
        else:
            invoc = sys.argv[0]
            if invoc.startswith("python"):
                invoc = f"{sys.argv[0]} {sys.argv[1]}"
            print(
                "Please first initialise this application by calling "
                f"`{invoc} config`."
            )
    return g_config

def get_model_state(config, table, key=None):
    url = f"{config['server']['url']}/api/{table}/"
    if key:
        url += f"{key}/"
    headers = {"Authorization": f"Token {config['server']['accessKey']}"}
    resp = requests.get(url, headers=headers)
    if not resp.ok:
        resp.raise_for_status()
    state = resp.json()
    return json.dumps(state)

def model_create(config, table, data):
    url = f"{config['server']['url']}/api/{table}/"
    headers = {"Authorization": f"Token {config['server']['accessKey']}"}
    resp = requests.post(url, data=data, headers=headers)
    if not resp.ok:
        resp.raise_for_status()
    state = resp.json()
    return json.dumps(state)

def print_json(json_str):
    parsed = json.loads(json_str)
    print(json.dumps(parsed, indent=4))

def notimplementedyet(placeholder_function):
    """Decorator method for unimplemented feature warnings"""

    def notimpl_version(*args, **kwargs):
        print(
            f"Error: The function {placeholder_function.__name__} is not yet "
            "implemented in the cli. Please use the web frontend to perform "
            "this action."
        )

    return notimpl_version


import os
import logging
import discord

from discord.ext import commands

# For logging events/errors
logger = logging.getLogger('discord')
logger.setLevel(logging.INFO)  # Options in decending order: CRITICAL, ERROR, WARNING, INFO, DEBUG; default WARNING
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

# Initialize the bot
bot = commands.Bot(command_prefix='?',
                   help_command=commands.DefaultHelpCommand(no_category='Misc', dm_help=True),
                   intents=discord.Intents.all())

# Load all extensions to start
for filename in os.listdir('./extensions'):
    if filename.endswith('.py'):
        bot.load_extension(f"extensions.{filename[:-3]}")

# Load the bot password (TOKEN) as an environment variable
TOKEN = os.getenv('BOT_TOKEN')

# Start the bot
bot.run(TOKEN)


import time
from configparser import ConfigParser, SectionProxy
from contextlib import suppress
from enum import Enum
from typing import Optional

from builder_const import Building, Config, BuildingCategory, BUILDING_BASE_URL
from selenium.common import NoSuchElementException
from selenium.webdriver.chrome.webdriver import WebDriver
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.webelement import WebElement
from termcolor import cprint
from utils import do_click, get_table_rows

class ExpansionStatus(Enum):
    to_build = 0
    done = 1
    in_progress = 2
    waiting_other = 3
    other = 4

class Expansions(str, Enum):
    rsd = "Rozbudowa o stanowisko dowodzenia"
    prisoner_van = "Rozbudowa o policyjne więźniarki"
    opp = "Rozbudowa o Oddział Prewencji Policji"
    wrd = "Rozbudowa o Wydział Ruchu Drogowego"
    prison_cells = "Cele"
    prison_cell_1 = "Cela więzienna"
    prison_cell_n = "Dodatkowa cela"
    pr = "Rozbudowa dla pojazdów proszkowych"
    containers = "Rozbudowa dla kontenerów"

def build_expansions(driver: WebDriver, building: Building, config: Config) -> None:
    driver.get(f"{BUILDING_BASE_URL}{building.id}")
    expansions_target_raw = config.ini[building.category.name]
    expansions_target = {k: _get_expansion_target_value(v) for k, v in expansions_target_raw.items()}

    do_click(driver, driver.find_element(By.XPATH, '//*[@id="tabs"]/li[2]/a'))
    time.sleep(0.3)

    current_expansions = get_expansions_status(driver)
    cprint(f'Current: {current_expansions}, target: {expansions_target}', 'yellow')
    to_build = {k: max(0, v - current_expansions.get(k, 0)) for k, v in expansions_target.items()}
    cprint(f'To build: {to_build}', 'yellow')

    while sum(to_build.values()) > 0:
        result = queue_expansions(driver, to_build)
        if not result:
            break

def _get_expansion_target_value(value: str) -> int:
    if value == 'False':
        return 0
    if value == 'True':
        return 1
    return int(value)

def _get_expansion(name: WebElement) -> Optional[Expansions]:
    with suppress(ValueError):
        parsed = name.find_element(By.TAG_NAME, 'b').get_attribute('innerHTML').strip()
        expansions = Expansions(parsed)
        if expansions == Expansions.prison_cell_n or expansions == expansions.prison_cell_1:
            return Expansions.prison_cells
    return None

def _get_status(actions: WebElement) -> ExpansionStatus:
    try:
        action_href = actions.find_element(By.TAG_NAME, 'a').get_attribute('href')
        if 'extension_cancel' in action_href:
            return ExpansionStatus.in_progress
        if 'extension_finish' in action_href:
            return ExpansionStatus.in_progress
        if 'extension_ready' in action_href:
            return ExpansionStatus.done
        if 'extension/credits' in action_href:
            return ExpansionStatus.to_build
    except NoSuchElementException:
        if "Pozostały czas" in actions.text:
            return ExpansionStatus.in_progress

    try:
        if "Pozostały czas" in actions.get_attribute('text'):
            return ExpansionStatus.in_progress
    except Exception as err:
        pass
    return ExpansionStatus.waiting_other

def get_expansions_status(driver: WebDriver) -> dict:
    rows = get_table_rows(driver, class_name="table")
    already_built = {}
    for row in rows:
        name, _, _, actions = row.find_elements(By.TAG_NAME, 'td')
        expansion = _get_expansion(name)
        if not expansion:
            continue

        expansion_status = _get_status(actions)
        if expansion_status in [ExpansionStatus.done, ExpansionStatus.in_progress]:
            already_built[expansion.name] = already_built.get(expansion.name, 0) + 1

    return already_built

def queue_expansions(driver: WebDriver, to_build: dict) -> bool:
    rows = get_table_rows(driver, class_name="table")
    for row in rows:
        name, _, _, actions = row.find_elements(By.TAG_NAME, 'td')
        expansion = _get_expansion(name)
        print(name, expansion)
        if not expansion:
            continue

        expansion_status = _get_status(actions)
        print(expansion_status, to_build.get(expansion.name), actions.text)
        if expansion_status is ExpansionStatus.to_build and to_build.get(expansion.name, 0) > 0:
            to_build[expansion.name] = to_build[expansion.name] - 1
            do_click(driver, actions.find_element(By.TAG_NAME, 'a'))
            cprint(f'Expansion was queued {expansion}', 'green')
            return True

    return False


# Giovanni Thsibangu
# Python Indeed Web Scraper
# 3/16/2022

import requests
from bs4 import BeautifulSoup
import pandas as pd
import csv
from datetime import *
import smtplib, ssl
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import smtplib
import mimetypes
from email import encoders
from email.message import Message
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
import time

# In[46]:

# Date and time object
today =date.today()
now =datetime.now().strftime("%H:%M:%S")

# Data frame and series Utilities
template  = {"Job Title":[],"Company Name":[],"Location":[],"Salaries":[],"Link":[]}
errorTemplate={"Eror":[],"Location":[],"Date":[],"Time":[]}
errorDataFrame= pd.DataFrame(errorTemplate)
Main_DataFrame =pd.DataFrame(template)

# Initiliaze variables
error_reporter=""
error_location=""
url=""
names=[]
titles=[]
links=[]
locations=[]
salaries=[]

# In[ ]:

# In[47]:

def getPage(url):

    try:
        page = requests.get(url)
        page.raise_for_status()
        soup = BeautifulSoup(page.content,'html.parser')
        result = soup.find(id="mosaic-provider-jobcards")
        job_elements = result.find_all("div", class_="job_seen_beacon")

        error_reporter="Success Soup"
        error_location="GetPage URL"
        errorLog_file(error_reporter,error_location,today,now)
        soup = BeautifulSoup(page.content,'html.parser')
        result = soup.find(id="mosaic-provider-jobcards")
        job_elements = result.find_all("div", class_="job_seen_beacon")

    except Exception as ex:

        error_reporter= str(ex)
        error_location="Soup"
        errorLog_file(error_reporter,error_location,today,now)
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)

        email_Update("error","getPage "+error_reporter)
        print("An error as occured in the Soup: ")
    else:
        print("Soup elements have been successfuly scraped.")
        return soup

def jobCard(url):
    try:
        soup =getPage(url)
    except Exception as ex:
        error_reporter=str(ex)
        error_location="getPage Function Failed"
        errorLog_file(error_reporter,error_location,today,now)
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        email_Update("error","jobCard "+error_reporter)
    else:
        title_element = soup.find_all("h2", class_="jobTitle")
        company_element = soup.find_all("span", class_="companyName")
        location_element = soup.find_all("div", class_="companyLocation")
        salary = soup.find_all("div", class_="attribute_snippet")
        job_link = soup.find_all("a", class_="tapItem")

        for title in title_element:
            titles.append(title.text)
        for name in company_element:
            names.append(name.text)
        for location in location_element:
            locations.append(location.text)
        for salar in salary:
            salaries.append(salar.text)
        for link in job_link:
            links.append("https://www.indeed.com"+link.get("href"))

        print("Success Job card  data")

        error_reporter="Success Jobcard"
        error_location="JobCard"
        errorLog_file(error_reporter,error_location,today,now)
        return (titles,names,locations,salaries,links)

def createDataFrame(url):
    try:
        jobCard(url)
        df_titles=pd.Series(titles,name="Job Title")
        df_names=pd.Series(names,name="Company Name")
        df_locations=pd.Series(locations,name="Location")
        df_salaries=pd.Series(salaries,name="Salaries")
        df_job_link=pd.Series(links,name="Links")
        # Frame contains all the row required for our dataFrame
        frames =[df_titles,df_names,df_locations,df_salaries,df_job_link]
        # Concat all.
        job_card =pd.concat(frames,axis=1)
        print("Data Frame successfull created")
    except Exception as ex:
        error_reporter=str(ex)
        error_location="createDataFrame"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        errorLog_file(error_reporter,error_location,today,now)
        email_Update("error","createDataFrame "+error_reporter)
        print(ex)
    else:
            # Return dataFrame
        return job_card
# In[54]:

# Email Function
def email_Update(file,subject):
    emailfrom = "<Your email>"
    emailto ="<Recipient email>"
    fileToSend = file+".csv"
    username = "user"
    password = "<Your password>"

    msg = MIMEMultipart()
    msg["From"] = emailfrom
    msg["To"] = emailto
    msg["Subject"] = subject
    msg.preamble = subject

    ctype, encoding = mimetypes.guess_type(fileToSend)
    if ctype is None or encoding is not None:
        ctype = "application/octet-stream"

    maintype, subtype = ctype.split("/", 1)

    if maintype == "text":
        fp = open(fileToSend)
        # Note: we should handle calculating the charset
        attachment = MIMEText(fp.read(), _subtype=subtype)
        fp.close()
    elif maintype == "image":
        fp = open(fileToSend, "rb")
        attachment = MIMEImage(fp.read(), _subtype=subtype)
        fp.close()
    elif maintype == "audio":
        fp = open(fileToSend, "rb")
        attachment = MIMEAudio(fp.read(), _subtype=subtype)
        fp.close()
    else:
        fp = open(fileToSend, "rb")
        attachment = MIMEBase(maintype, subtype)
        attachment.set_payload(fp.read())
        fp.close()
        encoders.encode_base64(attachment)
    attachment.add_header("Content-Disposition", "attachment", filename=fileToSend)
    msg.attach(attachment)

    server = smtplib.SMTP("smtp.gmail.com:587")
    server.starttls()
    server.login(emailfrom,password)
    server.sendmail(emailfrom, emailto, msg.as_string())
    print("Quitting the Server!!!")
    server.quit()
# In[55]:

def errorLog_file(error,loc,date,time):
    try:
        errorTemplate={"Eror":[error],"Location":[loc],"Date":[date],"Time":[time]}
        errorDataFrame= pd.DataFrame(errorTemplate)

    except Exception as ex:
        error_reporter=str(ex)
        error_location="Driver main"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)

        email_Update("error","ErrorLog_File "+error_reporter)
    else:
        errorDataFrame.to_csv("error.csv")
# In[56]:
def DriverMain(listOfposition):
    try:
        for job_title in listOfposition:
            city="waterbury"
            state="CT"
            url ="https://www.indeed.com/jobs?q={job_title}&l={city}%2C%20{state}&fromage=1&".format(job_title=job_title,city=city,state=state)

            file = createDataFrame(url)

            file.to_csv(job_title+".csv")
            email_Update(job_title,"Job Update!")
            print(url)
            print(job_title+" Job  CSV created Email sent!")
            time.sleep(10)
    except Exception as ex:
        print("Failed link: "+url)
        print(ex)
        error_reporter=str(ex)
        error_location="Driver main"
        message=error_reporter+" in the "+error_location+" Time "+str(today)+" "+str(now)
        email_Update("error","DriverMain "+error_reporter)
        errorLog_file(error_reporter,error_location,today,now)
    else:
        print(job_title," Job file has been searched and saved as csv ")
        print()
# In[59]:

listOfposition=["Developer","Programmer","software"] # List of job titles to search
DriverMain(listOfposition)      # Driver function


import json
import sys
from collections import defaultdict
from optparse import make_option
from urllib.parse import urlparse

import ckanapi
import pandas as pd
import lxml
from django.db.models import Count

from . import InventoryCommand
from inventory.models import Dataset, Distribution
from inventory.scrapers import CKAN

class Command(InventoryCommand):
    help = 'Analyzes catalogs'

    option_list = InventoryCommand.option_list + (
        make_option('--access', action='append_const', dest='reports', const='access',
                    help='Classification of direct download domain names, for import into R.'),
        make_option('--api', action='append_const', dest='reports', const='api',
                    help='Usage of catalog API technologies.'),
        make_option('--dcat', action='append_const', dest='reports', const='dcat',
                    help='Usage of DCAT by CKAN catalogs.'),
        make_option('--pod', action='append_const', dest='reports', const='pod',
                    help='Usage of Project Open Data Metadata Schema.'),
        make_option('--schemaorg', action='append_const', dest='reports', const='schemaorg',
                    help='Usage of Schema.org.'),
        make_option('--federation', action='append_const', dest='reports', const='federation',
                    help='Usage of Federation technologies.'),
        make_option('--licenses', action='append_const', dest='reports', const='licenses',
                    help='Usage of licenses.'),
        make_option('--csv', action='store_const', dest='format', const='csv',
                    default='table',
                    help='Prints the results as CSV.'),
    )

    def handle(self, *args, **options):
        self.setup(*args, **options)

        for report in options['reports']:
            result = getattr(self, report)()
            if result is not None:
                if options['format'] == 'table':
                    result.to_string(sys.stdout)
                elif options['format'] == 'csv':
                    result.to_csv(sys.stdout)

    def series(self, getter):
        series = {}
        for catalog in self.catalogs:
            series[catalog.division_id] = getter(catalog)
        return pd.Series(series)

    def api(self):
        def getter(catalog):
            if issubclass(catalog.scraper, CKAN):
                try:
                    client = ckanapi.RemoteCKAN(catalog.url, get_only=catalog.get_only)
                    status_show = client.call_action('status_show', verify=catalog.verify)
                    return int('datastore' in status_show['extensions'])
                except ckanapi.errors.CKANAPIError:
                    pass

        return self.series(getter)

    def dcat(self):
        def getter(catalog):
            if issubclass(catalog.scraper, CKAN):
                datasets = Dataset.objects.filter(division_id=catalog.division_id)
                if datasets.exists():
                    response = self.get(catalog.dataset_url(datasets[0]))
                    if response.status_code == 200:
                        response = self.get(catalog.dataset_rdf_url(datasets[0]))
                        return int(response.status_code == 200)

        return self.series(getter)

    def pod(self):
        def getter(catalog):
            response = self.get(catalog.data_json_url)
            return int(response.status_code == 200)

        return self.series(getter)

    def schemaorg(self):
        def getter(catalog):
            datasets = Dataset.objects.filter(division_id=catalog.division_id)
            if datasets.exists():
                url = catalog.dataset_url(datasets[0])
                if url:
                    response = self.get(url)
                    if response.status_code == 200:
                        return int('http://schema.org/Dataset' in response.text)

        return self.series(getter)

    def access(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            urls = Distribution.objects.filter(division_id=catalog.division_id).values_list('accessURL', flat=True)
            if urls:
                frame['count'][catalog.division_id] = urls.count()
                for url in urls:
                    host = urlparse(url).netloc.split(':', 1)[0]
                    if host.endswith(catalog_domains[catalog.division_id]):
                        key = 'Catalog'
                    elif any(host.endswith(suffix) for suffix in government_domains[catalog.division_id]):
                        key = 'Government'
                    else:
                        key = 'Other'
                        # frame[host][catalog.division_id] += 1
                    frame[key][catalog.division_id] += 1
        return pd.DataFrame(frame)

    def federation(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            # Assumes we don't need to paginate.
            if issubclass(catalog.scraper, CKAN):
                client = ckanapi.RemoteCKAN(catalog.url, get_only=catalog.get_only)
                package_search = client.call_action('package_search', {'fq': 'type:harvest', 'rows': 300000}, verify=catalog.verify)

                if package_search['results']:
                    for package in package_search['results']:
                        source_type = self.source_type(catalog, package)
                        if source_type:
                            frame[source_type][catalog.division_id] += 1
                        else:
                            self.warning('could not determine source type of {}'.format(catalog.dataset_api_url(package)))
                # GB
                else:
                    try:
                        for package in client.call_action('harvest_source_list', verify=catalog.verify):
                            if package['active']:
                                source_type = normalize_source_type(package, package['type'])
                                if source_type:
                                    frame[source_type][catalog.division_id] += 1
                                else:
                                    self.warning('could not determine source type of {}'.format(catalog.harvest_api_url(package)))
                    except ckanapi.errors.CKANAPIError:
                        pass
            elif catalog.scraper.__name__ == 'Socrata':
                if 'federation_filter' in self.get(catalog.url).text:
                    frame['socrata'][catalog.division_id] = 1
        return pd.DataFrame(frame)

    def licenses(self):
        frame = defaultdict(lambda: defaultdict(int))
        for catalog in self.catalogs:
            for value in Dataset.objects.filter(division_id=catalog.division_id).values('license').annotate(count=Count('id', distinct=True)).order_by('count').iterator():
                frame[catalog.division_id][value['license']] = value['count']
        return pd.DataFrame(frame)

    def report(self, klass, field, *, distinct):
        for catalog in self.catalogs:
            count = Dataset.objects.filter(division_id=catalog.division_id).count()
            print('{} ({})'.format(catalog.division_id, count))
            for value in klass.objects.filter(division_id=catalog.division_id).values(field).annotate(count=Count(distinct, distinct=True)).order_by('count').iterator():
                print('  {:7.2%} {} ({})'.format(value['count'] / count, value[field], value['count']))

    def source_type(self, catalog, package):
        # AU, FI, IE, IT, MX, PY
        if package.get('source_type'):
            return normalize_source_type(package, package['source_type'])

        # IT
        elif '/api/rest/dataset/' in package['url']:
            url, name = package['url'].split('api/rest/dataset/', 1)
            return self.source_type(catalog, ckanapi.RemoteCKAN(url).call_action('package_show', {'id': name}))

        # US
        # @see https://github.com/ckan/ckanext-spatial/blob/master/doc/harvesters.rst
        # @see https://github.com/GSA/ckanext-geodatagov/tree/master/ckanext/geodatagov/harvesters
        elif package.get('extras'):
            source_type = next(extra['value'] for extra in package['extras'] if extra['key'] == 'source_type')
            # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/base.py#L174
            if source_type == 'single-doc':
                response = self.get(package['url'])
                if response.status_code == 200:
                    try:
                        return normalize_metadata_scheme(response)
                    except lxml.etree.XMLSyntaxError:
                        pass
            # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/waf_collection.py
            elif source_type == 'waf-collection':
                # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/validation/__init__.py
                config = json.loads(next(extra['value'] for extra in package['extras'] if extra['key'] == 'config'))
                if config.get('validator_profiles'):
                    if len(config['validator_profiles']) > 1:
                        self.warning('multiple validator_profiles for {}'.format(catalog.dataset_api_url(package)))
                    else:
                        return 'waf-{}'.format(validators[config['validator_profiles'][0]])
                else:
                    response = self.get(config['collection_metadata_url'])
                    if response.status_code == 200:
                        scheme = normalize_metadata_scheme(response)
                        if scheme:
                            return 'waf-{}'.format(scheme)
            else:
                normalized = normalize_source_type(package, source_type)
                if normalized:
                    return normalized

        # BR
        else:
            try:
                if ckanapi.RemoteCKAN(package['url']).call_action('site_read'):
                    return 'ckan'
            except ckanapi.errors.CKANAPIError:
                pass

def normalize_source_type(package, source_type):
    if source_type in source_types:
        return source_types[source_type]
    elif package['url'].endswith('/csw') or '/csw/' in package['url']:
        return 'csw'

def normalize_metadata_scheme(response):
    if 'FGDC-STD-001-1998' in response.text:
        return 'fgdc'
    elif lxml.etree.fromstring(response.content).xpath('/MD_Metadata|/gmi:MI_Metadata', namespaces={'gmi': 'http://www.isotc211.org/2005/gmi'}):
        return 'iso19139'

source_types = {
    # Dynamic API
    'arcgis': 'arcgis',
    'ckan': 'ckan',
    'csw': 'csw',
    'waf': 'waf',
    # GB
    'gemini-waf': 'waf-gemini',
    # US
    # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/base.py#L185
    'geoportal': 'csw',
    # @see https://github.com/GSA/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/z3950.py
    'z3950': 'iso23950',

    # Static file
    # GB
    'dcat_rdf': 'dcat_rdf',
    'data_json': 'dcat_json',
    'gemini-single': 'gemini',
    'inventory': 'datashare',
    # MX
    'dcat_json': 'pod',  # v1.0
    # US
    'datajson': 'pod',
}

validators = {
    'fgdc_minimal': 'fgdc',
    'iso19139ngdc': 'iso19139',
}

catalog_domains = {
    'ocd-division/country:ar': 'datospublicos.gob.ar',
    'ocd-division/country:au': 'data.gov.au',
    'ocd-division/country:br': 'dados.gov.br',
    'ocd-division/country:ca': 'data.gc.ca',
    'ocd-division/country:cl': 'datos.gob.cl',
    'ocd-division/country:cr': 'gobiernodigitalcr.cloudapi.junar.com',
    'ocd-division/country:ee': 'opendata.riik.ee',
    'ocd-division/country:es': 'datos.gob.es',
    'ocd-division/country:fi': 'avoindata.fi',
    'ocd-division/country:fr': 'data.gouv.fr',
    'ocd-division/country:gb': 'data.gov.uk',
    'ocd-division/country:gh': 'data.gov.gh',
    'ocd-division/country:gr': 'data.gov.gr',
    'ocd-division/country:id': 'data.id',
    'ocd-division/country:ie': 'data.gov.ie',
    'ocd-division/country:it': 'dati.gov.it',
    'ocd-division/country:ke': 'opendata.go.ke',
    'ocd-division/country:md': 'data.gov.md',
    'ocd-division/country:mx': 'datos.gob.mx',
    'ocd-division/country:nl': 'data.overheid.nl',
    'ocd-division/country:ph': 'data.gov.ph',
    'ocd-division/country:py': 'datos.gov.py',
    'ocd-division/country:ro': 'data.gov.ro',
    'ocd-division/country:se': 'oppnadata.se',
    'ocd-division/country:sk': 'data.gov.sk',
    'ocd-division/country:tz': 'opendata.go.tz',
    'ocd-division/country:us': 'data.gov',
    'ocd-division/country:uy': 'catalogodatos.gub.uy',
}

# We only categorize the most popular domains, i.e. >=0.1% of distributions.
government_domains = {
    'ocd-division/country:ar': ['.gob.ar'],
    'ocd-division/country:au': ['.gov.au'],
    'ocd-division/country:br': [
        '.gov.br',
        '.jus.br',
        '.leg.br',
    ],
    'ocd-division/country:ca': [
        '.gc.ca',
        'geobase.ca',  # Natural Resources Canada
    ],
    'ocd-division/country:cl': ['.gob.cl'],
    'ocd-division/country:cr': ['.go.cr'],
    'ocd-division/country:ee': [
        '.riik.ee',  # "state"
        '.muinas.ee',  # http://et.wikipedia.org/wiki/Muinsuskaitseamet
    ],
    'ocd-division/country:es': [
        '.gob.es',
        '.cnig.es',  # Centro Nacional de Información Geográfica
        '.chj.es',  #  Confederación Hidrográfica del Júcar
        '.agenciatributaria.es',  # Agencia Tributaria
        '.fega.es',  # Fondo Español de Garantía Agraria
        '.ign.es',  #  Instituto Geográfico Nacional
        '.imserso.es',  # Instituto de Mayores y Servicios Sociales
        '.ine.es',  # Instituto Nacional de Estadística
        '.ipyme.org',  # Dirección General de Industria y de la Pequeña y Mediana Empresa
        '.cis.es',  # Centro de investigaciones sociológicas
    ],
    'ocd-division/country:fi': [  # 0.005
        '.gtk.fi',  # Geological Survey of Finland
        '.liikennevirasto.fi',  # Finnish Transport Agency
        '.maanmittauslaitos.fi',  # National Land Survey
        '.paikkatietoikkuna.fi',  # National Land Survey
        '.sfs.fi',  # Finnish Standards Association
        '.stat.fi',  # Statistics Finland
        '.vahtiohje.fi',  # Ministry of Finance
    ],
    'ocd-division/country:fr': ['.gouv.fr'],
    'ocd-division/country:gb': [
        '.gov.uk',
        '.hpa.org.uk',  # Public Health England
        '.isdscotland.org',  # Scottish Government
        '.nhs.uk',  # Department of Health
        '.ordnancesurvey.co.uk',  # Department for Business, Innovation and Skills
        '.slc.co.uk',  # http://en.wikipedia.org/wiki/Student_Loans_Company
        '.uktradeinfo.com',  # HM Revenue and Customs
        'opendatacommunities.org',  # Department for Communities and Local Government
    ],
    'ocd-division/country:gh': ['.gov.gh'],
    'ocd-division/country:gr': [  # 0.005
        '.gov.gr',
        '.astynomia.gr',  # http://en.wikipedia.org/wiki/Hellenic_Police
        '.ekdd.gr',  # http://en.wikipedia.org/wiki/National_Centre_for_Public_Administration_and_Local_Government
    ],
    'ocd-division/country:id': ['.go.id'],
    'ocd-division/country:ie': [  # 0.005
        '.gov.ie',
        '.marine.ie',  # Marine Institute
        '.cso.ie',  # Central Statistics Office Ireland
        '.hea.ie',  # Higher Education Authority
        '.buildingsofireland.ie',  # Department of Arts, Heritage and the Gaeltacht
        '.epa.ie',  # Environmental Protection Agency
        '.education.ie',  # Department of Education and Skills
        '.environ.ie',  # Department of the Environment, Community and Local Government
        '.fishingnet.ie',  # Department of Agriculture, Food and the Marine
        '.gsi.ie',  # Geological Survey of Ireland
        '.infomar.ie',  # Geological Survey of Ireland / Marine Institute
        '.seai.ie',  # Sustainable Energy Authority
    ],
    'ocd-division/country:it': [
        '.gov.it',
        '.inail.it',  # Istituto Nazionale Assicurazione contro gli Infortuni sul Lavoro
        '.inps.it',  # Istituto Nazionale Previdenza Sociale
        '.istat.it',  # Istituto nazionale di statistica
        '.politicheagricole.it',  # Ministero delle politiche agricole alimentari e forestali
    ],
    'ocd-division/country:ke': [
        '.go.ke',
        '.kenyalaw.org',  # National Council for Law Reporting
    ],
    'ocd-division/country:md': [
        '.gov.md',
        '.statistica.md',  # Biroul Naţional de Statistică al Republicii Moldova
        '.knbs.or.ke',  # Kenya National Bureau of Statistics
    ],
    'ocd-division/country:mx': [
        '.gob.mx',
        '.nafin.com',  # Nacional Financiera, state bank
        '.pemex.com',  # Petróleos Mexicanos, state-owned
    ],
    'ocd-division/country:nl': [
        '.duo.nl',  # Ministerie van Onderwijs, Cultuur en Wetenschap
        '.gdngeoservices.nl',  # Geologische Dienst Nederland
        '.kaartenbalie.nl',  # Ministerie van Economische Zaken
        '.kadaster.nl',  # Cadastre, Land Registry and Mapping Agency
        '.knmi.nl',  # Ministry of Infrastructure and the Environment
        '.nationaalgeoregister.nl',
        '.overheid.nl', # "government"
        '.pbl.nl',  # Planbureau voor de Leefomgeving
        '.rijkswaterstaat.nl',  # Ministerie van Infrastructuur en Milieu
        '.risicokaart.nl',
        '.rivm.nl',  # Ministerie van Volksgezondheid, Welzijn en Sport
        '.rwsgeoweb.nl',
        '.waterschapservices.nl',
    ],
    'ocd-division/country:ph': ['.gov.ph'],
    'ocd-division/country:py': ['.gov.py'],
    'ocd-division/country:ro': [  # 0.005
        '.gov.ro',
        '.edu.ro',  # Ministerul Educației Naționale
        '.inforegio.ro',  # Ministry of Regional Development and Public Administration
        '.mfinante.ro',  # Ministerul Finanţelor Publice
        '.mmuncii.ro',  # Ministerul Muncii, Familiei, Protecţiei Sociale şi Persoanelor Vârstnice
    ],
    'ocd-division/country:se': [  # 0.025
        '.gov.se',
        '.sgu.se',  # http://en.wikipedia.org/wiki/Geological_Survey_of_Sweden
        '.socialstyrelsen.se',  # http://en.wikipedia.org/wiki/National_Board_of_Health_and_Welfare_%28Sweden%29
        '.verksamt.se',
    ],
    'ocd-division/country:sk': [  # 0.005
        '.gov.sk',
        '.justice.sk',  # Ministerstvo Spravodlivosti
        '.mfa.sk',  # Ministry of Foreign and European Affairs of the Slovak Republic
        '.mhsr.sk',  # Ministry of Economy of the Slovak Republic
        '.minv.sk',  # Ministry of Interior of the Slovak Republic
        '.mosr.sk',  # Ministry of Defense of Slovak Republic
        '.mpsr.sk',  # Ministry of Agriculture and Rural Development of the Slovak Republic
        '.nrsr.sk',  # National Council of the Slovak Republic
        '.sazp.sk',  #  Slovenská agentúra životného prostredia
        '.skgeodesy.sk',  # Geodesy, Cartography and Cadastre Authority of Slovak Republic
        '.statistics.sk',  # Statistical Office of the Slovak Republic
        '.svssr.sk',  # State Veterinary and Food Administration of the Slovak Republic
    ],
    'ocd-division/country:tz': ['.go.tz'],
    'ocd-division/country:us': [
        '.gov',
        '.mil',
    ],
    'ocd-division/country:uy': [
        '.gub.uy',
        '.precios.uy',  #  Ministerio de Economía y Finanzas
    ],
}


import torch
import torch.nn as nn
from torch.autograd import Variable

# CUDA settings
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class ConvLSTMCell(nn.Module):
    def __init__(self, input_channels, hidden_channels, kernel_size,
                 weight_dict = None, cell_index = None):
        """
        Build convolutional cell for ConvLSTM.
        param input_channels: number of channels (variables) from input fields
        param hidden_channels: number of channels inside hidden layers, the dimension correponds to the output size
        param kernel_size: size of filter, if not a square then need to input a tuple (x,y)
        param weight_dict: weight matrix for the initialization of mu (mean)
        param cell_index: index of created BayesConvLSTM cell
        """
        super(ConvLSTMCell, self).__init__()

        #assert hidden_channels % 2 == 0

        self.input_channels = input_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        #self.num_features = 4

        # inherit weight matrix
        self.weight_dict = weight_dict
        self.cell_index = cell_index

        self.padding = int((kernel_size - 1) / 2) # make sure the output size remains the same as input
        # input shape of nn.Conv2d (input_channels,out_channels,kernel_size, stride, padding)
        # kernal_size and stride can be tuples, indicating non-square filter / uneven stride
        self.Wxi = nn.Conv2d(self.input_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=True)
        self.Whi = nn.Conv2d(self.hidden_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=False)
        self.Wxf = nn.Conv2d(self.input_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=True)
        self.Whf = nn.Conv2d(self.hidden_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=False)
        self.Wxc = nn.Conv2d(self.input_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=True)
        self.Whc = nn.Conv2d(self.hidden_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=False)
        self.Wxo = nn.Conv2d(self.input_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=True)
        self.Who = nn.Conv2d(self.hidden_channels, self.hidden_channels, self.kernel_size, 1, self.padding, bias=False)

        self.Wci = None
        self.Wcf = None
        self.Wco = None

        if self.weight_dict is not None:
            # weight
            self.Wxi.weight.data = self.weight_dict['cell{}.Wxi.weight'.format(self.cell_index)].data
            self.Whi.weight.data = self.weight_dict['cell{}.Whi.weight'.format(self.cell_index)].data
            self.Wxf.weight.data = self.weight_dict['cell{}.Wxf.weight'.format(self.cell_index)].data
            self.Whf.weight.data = self.weight_dict['cell{}.Whf.weight'.format(self.cell_index)].data
            self.Wxc.weight.data = self.weight_dict['cell{}.Wxc.weight'.format(self.cell_index)].data
            self.Whc.weight.data = self.weight_dict['cell{}.Whc.weight'.format(self.cell_index)].data
            self.Wxo.weight.data = self.weight_dict['cell{}.Wxo.weight'.format(self.cell_index)].data
            self.Who.weight.data = self.weight_dict['cell{}.Who.weight'.format(self.cell_index)].data

            # bias
            self.Wxi.bias.data = self.weight_dict['cell{}.Wxi.bias'.format(self.cell_index)].data
            self.Wxf.bias.data = self.weight_dict['cell{}.Wxf.bias'.format(self.cell_index)].data
            self.Wxc.bias.data = self.weight_dict['cell{}.Wxc.bias'.format(self.cell_index)].data
            self.Wxo.bias.data = self.weight_dict['cell{}.Wxo.bias'.format(self.cell_index)].data

    def forward(self, x, h, c):
        ci = torch.sigmoid(self.Wxi(x) + self.Whi(h) + c * self.Wci)
        cf = torch.sigmoid(self.Wxf(x) + self.Whf(h) + c * self.Wcf)
        cc = cf * c + ci * torch.tanh(self.Wxc(x) + self.Whc(h))
        co = torch.sigmoid(self.Wxo(x) + self.Who(h) + cc * self.Wco)
        ch = co * torch.tanh(cc)
        return ch, cc

    def init_hidden(self, batch_size, hidden, shape):
        if self.Wci is None:
            self.Wci = Variable(torch.zeros(1, hidden, shape[0], shape[1])).to(device)
            self.Wcf = Variable(torch.zeros(1, hidden, shape[0], shape[1])).to(device)
            self.Wco = Variable(torch.zeros(1, hidden, shape[0], shape[1])).to(device)
        else:
            assert shape[0] == self.Wci.size()[2], 'Input Height Mismatched!'
            assert shape[1] == self.Wci.size()[3], 'Input Width Mismatched!'
        return (Variable(torch.randn(batch_size, hidden, shape[0], shape[1])).to(device),
                Variable(torch.randn(batch_size, hidden, shape[0], shape[1])).to(device))

class ConvLSTM(nn.Module):
    """
    This is the main ConvLSTM module.
    param input_channels: number of channels (variables) from input fields
    param hidden_channels: number of channels inside hidden layers, for multiple layers use tuple, the dimension correponds to the output size
    param kernel_size: size of filter, if not a square then need to input a tuple (x,y)
    param step: this parameter indicates the time step to predict ahead of the given data
    param effective_step: this parameter determines the source of the final output from the chosen step
    """
    # input_channels corresponds to the first input feature map
    # hidden state is a list of succeeding lstm layers.
    def __init__(self, input_channels, hidden_channels, kernel_size, weight_dict = None):
        super(ConvLSTM, self).__init__()
        self.input_channels = [input_channels] + hidden_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.num_layers = len(hidden_channels)
        #self.step = step
        #self.effective_step = effective_step
        self._all_layers = []
        for i in range(self.num_layers):
            name = 'cell{}'.format(i)
            if weight_dict is None:
                cell = ConvLSTMCell(self.input_channels[i], self.hidden_channels[i], self.kernel_size)
            else:
                cell = ConvLSTMCell(self.input_channels[i], self.hidden_channels[i], self.kernel_size,
                                   weight_dict, i)
            setattr(self, name, cell)
            self._all_layers.append(cell)

    def forward(self, x, timestep):
        """
        Forward module of ConvLSTM.
        param x: input data with dimensions [batch size, channel, height, width]
        """
        if timestep == 0:
            self.internal_state = []
        # loop inside
        for i in range(self.num_layers):
            # all cells are initialized in the first step
            name = 'cell{}'.format(i)
            if timestep == 0:
                #print('Initialization')
                bsize, _, height, width = x.size()
                (h, c) = getattr(self, name).init_hidden(batch_size=bsize, hidden=self.hidden_channels[i],
                                                         shape=(height, width))
                self.internal_state.append((h, c))

            # do forward
            (h, c) = self.internal_state[i]
            x, new_c = getattr(self, name)(x, h, c)
            self.internal_state[i] = (x, new_c)
            # only record output from last layer
            if i == (self.num_layers - 1):
                outputs = x

        return outputs, (x, new_c)

if __name__ == '__main__':
    # gradient check
    convlstm = ConvLSTM(input_channels=512, hidden_channels=[128, 64, 64, 32, 32], kernel_size=3).to(device)
    loss_fn = torch.nn.MSELoss()

    input = Variable(torch.randn(1, 512, 64, 32)).to(device)
    target = Variable(torch.randn(1, 32, 64, 32)).double().to(device)

    output = convlstm(input)
    output = output[0][0].double()
    res = torch.autograd.gradcheck(loss_fn, (output, target), eps=1e-6, raise_exception=True)
    print(res)