"""A class for methods to handle Main Page """

import logging
from time import sleep
from selenium.common.exceptions import *
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from Modules.BasePage.BasePage import BasePage
from Modules.load_class import LoadClass

class MainPage(BasePage):

    def dismiss_notifications(self):

        self.switch_context_to_webview()

        logging.info("dismiss notifications")
        try:
            notification_msg_on = self.driver.find_element(*self.configuration.LoginScreen.
                                                           NOTIFICATION_ABOUT_SENDING_MESSAGES)
            if notification_msg_on.is_displayed():
                logging.info("click 'No' for sending notifications")
                notification_msg_on = self.driver.find_element(
                    *self.configuration.LoginScreen.NOTIFICATION_ABOUT_SENDING_MESSAGES)
                self.assertIsNotNone(notification_msg_on, "Notification msg not found")
                self.driver.find_element(*self.configuration.LoginScreen.NO_FOR_SENDING_NOTIFICATIONS_ON).click()
            else:
                pass
        except NoSuchElementException:
            logging.info("notifications alert not present")

        self.switch_context_to_native()

    def alert_expiring_password(self):

        logging.info("check if Notice alert, about expiring password, is present")
        try:
            self.driver.find_element(*self.configuration.LoginScreen.ALERT_MSG_WILL_EXPIRE)
            self.assertIsNotNone(*self.configuration.LoginScreen.ALERT_MSG_WILL_EXPIRE)
            logging.info("Notice alert is present")
            self.driver.find_element(*self.configuration.LoginScreen.OK_BUTTON).click()
        except NoSuchElementException:
            logging.info("Notice alert is not present")
            pass

    def check_presence_of_events_button(self):

        # self.switch_context_to_webview()  # it looks like in webview all elements are present everywhere
        # and wait for condition is not working correctly in webview

        WebDriverWait(self.driver, 30).until(
            expected_conditions.presence_of_element_located(self.configuration.MainMenuScreen.EVENTS_BUTTON),
            "Events button in Main Menu is not present")

        # self.switch_context_to_native()

    def check_presence_of_inbox_button(self):

        self.switch_context_to_native()

        WebDriverWait(self.driver, 30).until(
            expected_conditions.presence_of_element_located(self.configuration.MainMenuScreen.INBOX_BUTTON),
            "Inbox button in Main Menu is not present")

    def click_ACTIVATE_BUTTON_on_alert(self):

        logging.info("click ACTIVATE button on alert")
        alert_activate_button = self.driver.find_element(*self.configuration.MainMenuScreen.ALERT_ACTIVATE_BUTTON)
        self.assertIsNotNone(alert_activate_button, "ACTIVATE button on alert not found")
        alert_activate_button.click()
        sleep(2)

    def check_if_alert_WORKFLOW_ACTIVATED_is_present(self):

        logging.info("check if alert 'Workflow activated' is present and click 'Ok'")
        WebDriverWait(self.driver, 25).until(
            expected_conditions.presence_of_element_located(self.configuration.MainMenuScreen.ALERT_WORKFLOW_ACTIVATED),
            "Events button in Main Menu is not present")
        self.driver.find_element(*self.configuration.LoginScreen.OK_BUTTON).click()

    def click_ACTIVATE_WORKFLOW(self):

        self.switch_context_to_webview()

        sleep(1)
        logging.info("click ACTIVATE WORKFLOW button")
        activate_workflow_button = self.driver.find_element(*self.configuration.MainMenuScreen.ACTIVATE_WORKFLOW_BUTTON)
        self.assertIsNotNone(activate_workflow_button, "ACTIVATE WORKFLOW button not found")
        activate_workflow_button.click()

        self.switch_context_to_native()

    def open_CREATE_CONTACT(self):

        self.switch_context_to_webview()

        logging.info("click CREATE CONTACT button")
        create_contact_button = self.driver.find_element(*self.configuration.MainMenuScreen.CREATE_CONTACT_BUTTON)
        self.assertIsNotNone(create_contact_button, "CREATE CONTACT button not found")
        create_contact_button.click()

        self.switch_context_to_native()

    def open_CREATE_TASK(self):

        self.switch_context_to_webview()

        logging.info("click CREATE TASK button")
        create_task_button = self.driver.find_element(*self.configuration.MainMenuScreen.CREATE_TASK_BUTTON)
        self.assertIsNotNone(create_task_button, "CREATE TASK button not found")
        create_task_button.click()

        self.switch_context_to_native()

    def open_CREATE_REPORT(self):

        self.switch_context_to_webview()

        logging.info("click CREATE REPORT button")
        create_report_button = self.driver.find_element(*self.configuration.MainMenuScreen.CREATE_REPORT_BUTTON)
        self.assertIsNotNone(create_report_button, "CREATE REPORT button not found")
        create_report_button.click()

        self.switch_context_to_native()

    def open_WEBSITE_LINK(self):

        logging.info("click WEBSITE LINK button")
        website_link_button = self.driver.find_element(*self.configuration.MainMenuScreen.WEBSITE_LINK_BUTTON)
        self.assertIsNotNone(website_link_button, "WEBSITE LINK button not found")
        website_link_button.click()

    def open_INCIDENT(self):

        self.switch_context_to_webview()

        sleep(1)
        logging.info("click INCIDENT button")
        incident_button = self.driver.find_element(*self.configuration.MainMenuScreen.INCIDENT_BUTTON)
        self.assertIsNotNone(incident_button, "INCIDENT button not found")
        incident_button.click()

        self.switch_context_to_native()

    def open_CREATE_ASSETS(self):

        self.switch_context_to_webview()

        logging.info("click CREATE ASSETS button")
        create_assets_button = self.driver.find_element(*self.configuration.MainMenuScreen.CREATE_ASSETS_BUTTON)
        self.assertIsNotNone(create_assets_button, "CREATE ASSETS button not found")
        create_assets_button.click()

        self.switch_context_to_native()

    def open_CREATE_LOG(self):

        self.switch_context_to_webview()

        logging.info("click CREATE LOG button")
        create_log_button = self.driver.find_element(*self.configuration.MainMenuScreen.CREATE_LOG_BUTTON)
        self.assertIsNotNone(create_log_button, "CREATE LOG button not found")
        create_log_button.click()

        self.switch_context_to_native()

    def open_RISKS(self):

        self.switch_context_to_webview()

        logging.info("click RISKS button")
        risks_button = self.driver.find_element(*self.configuration.MainMenuScreen.RISKS_BUTTON)
        self.assertIsNotNone(risks_button, "RISKS button not found")
        risks_button.click()

        self.switch_context_to_native()

        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_EVENTS(self):

        self.switch_context_to_webview()

        logging.info("click Events button")
        events_button = self.driver.find_element(*self.configuration.MainMenuScreen.EVENTS_BUTTON)
        self.assertIsNotNone(events_button, "EVENTS button not found")
        events_button.click()
        # logging.info("check if Events were opened")
        # events_header = self.driver.find_element(*self.configuration.EventsScreen.EVENTS_HEADER)
        # self.assertIsNotNone(events_header)

        self.switch_context_to_native()

        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_LOGS(self):

        self.switch_context_to_webview()

        logging.info("click LOGS button")
        logs_button = self.driver.find_element(*self.configuration.MainMenuScreen.LOGS_BUTTON)
        self.assertIsNotNone(logs_button, "LOGS button not found")
        logs_button.click()

        self.switch_context_to_native()

        # sleep(1)
        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_REPORTS(self):

        self.switch_context_to_webview()

        logging.info("click REPORTS button")
        reports_button = self.driver.find_element(*self.configuration.MainMenuScreen.REPORTS_BUTTON)
        self.assertIsNotNone(reports_button, "REPORTS button not found")
        reports_button.click()

        self.switch_context_to_native()

        # sleep(1)
        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_MAP(self):

        self.switch_context_to_webview()

        logging.info("click Map button")
        map_button = self.driver.find_element(*self.configuration.MainMenuScreen.MAP_BUTTON)
        self.assertIsNotNone(map_button, "MAP button not found")
        map_button.click()

        self.switch_context_to_native()

    def open_LOCATION(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_LOCATION()

    def open_ASSETS(self):

        self.switch_context_to_webview()

        logging.info("click ASSETS button")
        assets_button = self.driver.find_element(*self.configuration.MainMenuScreen.ASSETS_BUTTON)
        self.assertIsNotNone(assets_button, "ASSETS button not found")
        assets_button.click()

        self.switch_context_to_native()

        # sleep(1)
        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_MY_MESSAGES(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_MY_MESSAGES()

    def open_INBOX(self):

        self.switch_context_to_webview()

        logging.info("click INBOX button")
        inbox_button = self.driver.find_element(*self.configuration.MainMenuScreen.INBOX_BUTTON)
        self.assertIsNotNone(inbox_button, "INBOX button not found")
        inbox_button.click()
        # sleep(2)

        self.switch_context_to_native()

        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_COMPOSE(self):

        self.switch_context_to_webview()

        logging.info("click COMPOSE button")
        compose_button = self.driver.find_element(*self.configuration.MainMenuScreen.COMPOSE_BUTTON)
        self.assertIsNotNone(compose_button, "COMPOSE button not found")
        compose_button.click()

        self.switch_context_to_native()

    def open_SENT(self):

        self.switch_context_to_webview()

        logging.info("click SENT button")
        sent_button = self.driver.find_element(*self.configuration.MainMenuScreen.SENT_BUTTON)
        self.assertIsNotNone(sent_button, "SENT button not found")
        sent_button.click()

        self.switch_context_to_native()

        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_PHOTO(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_PHOTO()

    def open_VIDEO(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_VIDEO()

    def open_SOUND(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_SOUND()

    def open_TASKS(self):

        self.switch_context_to_webview()

        logging.info("click TASKS button")
        tasks_button = self.driver.find_element(*self.configuration.MainMenuScreen.TASKS_BUTTON)
        self.assertIsNotNone(tasks_button, "TASKS button not found")
        tasks_button.click()

        self.switch_context_to_native()

        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_DOCUMENTS(self):

        self.switch_context_to_webview()

        logging.info("click DOCUMENTS button")
        documents_button = self.driver.find_element(*self.configuration.MainMenuScreen.DOCUMENTS_BUTTON)
        self.assertIsNotNone(documents_button, "DOCUMENTS button not found")
        documents_button.click()

        self.switch_context_to_native()

    def open_CONTACTS(self):

        self.switch_context_to_webview()

        logging.info("click CONTACTS button")
        contacts_button = self.driver.find_element(*self.configuration.MainMenuScreen.CONTACTS_BUTTON)
        self.assertIsNotNone(contacts_button, "CONTACTS button not found")
        contacts_button.click()

        self.switch_context_to_native()

        sleep(1)
        common_page = LoadClass.load_page('CommonPage')
        common_page.setDriver(self.driver)
        common_page.wait_for_app_loading()

    def open_ALLOCATE(self):

        self.switch_context_to_webview()

        logging.info("click ALLOCATE button")
        allocate_button = self.driver.find_element(*self.configuration.MainMenuScreen.ALLOCATE_BUTTON)
        self.assertIsNotNone(allocate_button, "ALLOCATE button not found")
        allocate_button.click()

        self.switch_context_to_native()

    def open_SETTINGS(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_SETTINGS()

    def open_ACTIVATE(self):

        self.switch_context_to_webview()

        logging.info("click ACTIVATE button")
        activate_button = self.driver.find_element(*self.configuration.MainMenuScreen.ACTIVATE_BUTTON)
        self.assertIsNotNone(activate_button, "ACTIVATE button not found")
        activate_button.click()

        self.switch_context_to_webview()

    def open_OFFLINE_SYNC(self):

        self.switch_context_to_webview()

        logging.info("click OFFLINE SYNC button")
        offline_sync_button = self.driver.find_element(*self.configuration.MainMenuScreen.OFFLINE_SYNC_BUTTON)
        self.assertIsNotNone(offline_sync_button, "OFFLINE SYNC button not found")
        offline_sync_button.click()

        self.switch_context_to_native()

    def open_ABOUT(self):

        welcome_page = LoadClass.load_page('WelcomePage')
        welcome_page.setDriver(self.driver)
        welcome_page.open_ABOUT()

    def click_logout_button(self):

        logging.info("click LOGOUT button")
        logout_button = self.driver.find_element(*self.configuration.MainMenuScreen.LOGOUT_BUTTON)
        self.assertIsNotNone(logout_button, "Logout button not found")
        logout_button.click()


#!/usr/bin/env python

"""
Markdown extension which maps all local filenames (i.e. file://) to a
subdirectory specified by the config variable "resource_dir". Relative paths are
presumed to be relative to "relative_path" or the current working directory if
not given.

Returns a list of (local_path, desired_path) in md.resources where local_path is
the path to the file on the local machine and desired_path is the new path.
"""

from markdown.treeprocessors import Treeprocessor
from markdown.extensions     import Extension

import os

from util import unique

class ResourceExtractorTreeprocessor(Treeprocessor):

	# A list of types of tags which point to resources and the attribute which
	# contains the address of the resource.
	RESOURCE_TAGS = {
		"img": "src",
		"a":   "href",
	}

	def __init__(self, md, configs):
		self.md = md
		self.configs = configs

	def remap_resources(self, root, local_file_paths = None, desired_file_names = None):
		local_file_paths   = local_file_paths   if local_file_paths   is not None else []
		desired_file_names = desired_file_names if desired_file_names is not None else []

		for child in root:
			if child.tag in ResourceExtractorTreeprocessor.RESOURCE_TAGS:
				attrib = ResourceExtractorTreeprocessor.RESOURCE_TAGS[child.tag]
				file_path = child.attrib[attrib]
				if file_path.startswith("file://"):
					local_file_path = os.path.join( self.configs.get("relative_path",".")
					                              , file_path[len("file://"):]
					                              )
					desired_file_name = unique(os.path.basename(local_file_path), desired_file_names)

					child.attrib[attrib] = "%s/%s"%(self.configs["resource_dir"],desired_file_name)

					local_file_paths.append(local_file_path)
					desired_file_names.append(desired_file_name)

			# Recurse
			self.remap_resources(child, local_file_paths, desired_file_names)

		# Return with the resource dirs prefixed
		return zip(local_file_paths,
		           ( "%s/%s"%(self.configs["resource_dir"], dfn)
		             for dfn in desired_file_names)
		           )

	def run(self, root):
		self.md.resources = self.remap_resources(root)
		return root

class ResourceExtractor(Extension):

	def __init__(self, configs):
		self.configs = dict(configs)

	def extendMarkdown(self, md, md_globals):
		md.treeprocessors.add(
			'resourceextractor',
			ResourceExtractorTreeprocessor(md, configs = self.configs),
			'_end'
		)

def makeExtension(**kwargs):
	return ResourceExtractor(kwargs.get("configs", {}))


import os
import shutil
import logging

from django.test import Client
from django.test import TestCase
from django.conf import settings
from django.test.utils import override_settings

from .models import InlineImage
from .models import HostedImage
from .exceptions import TooBigMedia
from .exceptions import InvalidMimeType

from munch.apps.users.tests.factories import UserFactory
from munch.apps.campaigns.tests.factories import MessageFactory

logging.disable(logging.CRITICAL)

class TestHostedMedia(TestCase):
    """ Deletes the media dir once tests are completed

    WARNING: use that only if you are in tests, with an overloaded  MEDIA_ROOT
    Normally it's fine within munch, defined in munch/test_settings.py
    """
    def classSetUp(self):
        # mktemp will only be used for the 1st Case using this parent class
        if not os.path.exists(settings.MEDIA_ROOT):
            os.mkdir(settings.MEDIA_ROOT)

    def classTearDown(self):
        try:
            shutil.rmtree(settings.MEDIA_ROOT)
        except Exception:
            pass

class TestHostedImage(TestHostedMedia):
    def setUp(self):
        self.user = UserFactory()

    def test_image_toobig(self):
        class SmallHostedImage(HostedImage):
            # very small
            MAX_SIZE = 100
        img_url = 'http://www.oasiswork.fr/uploads/logo_oasiswork.jpg'

        with self.assertRaises(TooBigMedia):
            img = SmallHostedImage(
                img_url, organization=self.user.organization)
            img.store()

    def test_wrong_mimetype(self):
        html_document = 'data:image/gif;base64,PGh0bWw+PC9odG1sPg=='
        image = InlineImage(html_document, organization=self.user.organization)
        with self.assertRaises(InvalidMimeType):
            image.store()

class TestHostedImageClient(TestHostedMedia):
    def setUp(self):
        self.client = Client()
        self.user = UserFactory()

    def test_fetch_hosted_image_ok(self):
        img_url = 'http://www.oasiswork.fr/uploads/logo_oasiswork.jpg'
        img = HostedImage(img_url, organization=self.user.organization)
        stored_url = img.store()
        self.assertTrue(stored_url.startswith('http://munch.example.com'))

    def test_fetch_inline_image_ok(self):
        img_data = (
            "data:image/gif;base64,R0lGODlhDwAPAKECAAAAzMzM/////"
            "wAAACwAAAAADwAPAAACIISPeQHsrZ5ModrLlN48CXF8m2iQ3YmmKqVlRtW4ML"
            "wWACH+H09wdGltaXplZCBieSBVbGVhZCBTbWFydFNhdmVyIQAAOw==")
        img = InlineImage(img_data, organization=self.user.organization)
        stored_url = img.store()
        self.assertTrue(stored_url.startswith('http://munch.example.com'))

@override_settings(SECRET_KEY='123412341234')
class TestHostedMessage(TestCase):
    def setUp(self):
        self.user = UserFactory()
        self.message = MessageFactory(author=self.user)

    def test_hosted_mail_without_key_works(self):
        response = self.client.get('/archive/{}/'.format(
            self.message.identifier))
        self.assertEqual(response.status_code, 200)

    def test_hosted_mail_bad_uuid(self):
        response = self.client.get('/archive/1584498-148484/')
        self.assertEqual(response.status_code, 404)


# -*- coding: utf-8 -*-
"""
URLs for Api Collection list app
"""

from django.conf.urls import url
from .views import ApiCollectionListView, ExportCsvView

urlpatterns = [
    url(r'^$',
        ApiCollectionListView.as_view(),
        name='apicollection-list'),
    url(r'^export_csv$',
        ExportCsvView.as_view(),
        name='export-csv-apicollection')
]


# -*- coding: utf-8 -*-
import os, sys, json, time, random, urllib
project_dir = os.path.dirname(os.path.abspath(os.path.curdir))
sys.path.append(
    project_dir
)
sys.path.append(
    os.path.join(project_dir, 'lib')
)
print(sys.path)

# import unittest, requests, mimetypes
import tornado_server as entry
# import asr_server.http_get_info as
from functools import partial
from uuid import uuid4
import tornado.ioloop
import unittest
import mimetypes

import tornado.httpclient
import tornado.ioloop
from tornado.testing import *

'''
POST / HTTP/1.1
Host: localhost:8000
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Connection: keep-alive
Upgrade-Insecure-Requests: 1
Content-Type: multipart/form-data; boundary=---------------------------8721656041911415653955004498
Content-Length: 465

-----------------------------8721656041911415653955004498
Content-Disposition: form-data; name="myTextField"

Test
-----------------------------8721656041911415653955004498
Content-Disposition: form-data; name="myCheckBox"

on
-----------------------------8721656041911415653955004498
Content-Disposition: form-data; name="myFile"; filename="test.txt"
Content-Type: text/plain

Simple file.
-----------------------------8721656041911415653955004498--
'''

@gen.coroutine
def multipart_producer(boundary, keyname, filenames, params, write):
    boundary_bytes = boundary.encode()

    for filename in filenames:
        filename_bytes = filename.encode()
        mtype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
        buf = (
            (b"--%s\r\n" % boundary_bytes)
            + (
                b'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'
                % (keyname.encode(), filename_bytes)
            )
            + (b"Content-Type: %s\r\n" % mtype.encode())
            + b"\r\n"
        )
        yield write(buf)
        with open(filename, "rb") as f:
            while True:
                # 16k at a time.
                chunk = f.read(16 * 1024)
                if not chunk:
                    break
                yield write(chunk)

        yield write(b"\r\n")
    for k, v in params.items():
        buf = (
                (b"--%s\r\n" % boundary_bytes)
                + (
                        b'Content-Disposition: form-data; name="%s"\r\n'
                        % (k.encode())
                )
                + b"\r\n"
        )
        yield write(buf)
        yield write(v.encode())
        yield write(b"\r\n")

    yield write(b"--%s--\r\n" % (boundary_bytes,))

class Badmusictest(AsyncHTTPTestCase):
    def setUp(self):
        super(Badmusictest, self).setUp()

    def get_app(self):
        # 设置参数
        self.port, self.start_num = "7894", 11
        args = ['--port', self.port, '--start_num', str(self.start_num)]
        old_sys_argv = sys.argv
        sys.argv = [old_sys_argv[0]] + args
        self.args = entry.getArg()
        return entry.make_app(self.args)

    def test_getArg(self):
        args = entry.getArg()
        self.assertEqual(args.port, self.port)
        self.assertEqual(args.start_num, self.start_num)

    def test_checkMusicHandler_POST(self):
        url = '/checkbadmusic'
        boundary = uuid4().hex
        headers = {"Content-Type": "multipart/form-data; boundary=%s" % boundary}
        params = {'vid': str(int(time.time()))+''.join(random.choice('abcdef') for _ in range(8))}
        producer = partial(multipart_producer, boundary, 'sound', ['test.wav'], params)
        response = self.fetch(
            path=url,
            method="POST",
            headers=headers,
            body_producer=producer,
        )
        self.assertEqual(response.code, 200)
        res = json.loads(response.body)
        self.assertEqual(res['error'], '')

    def test_checkMusicHandler_GET(self):
        url = '/checkbadmusic'
        params = {
            'vid': str(int(time.time()))+''.join(random.choice('abcdef') for _ in range(8)),
            'sound_url': 'https://file-examples.com/wp-content/uploads/2017/11/file_example_WAV_2MG.wav',
        }
        response = self.fetch(
            path=url,
            body=urllib.urlencode(params)
        )
        self.assertEqual(response.code, 200)

import HtmlTestRunner

if __name__ == "__main__":
    runner = HtmlTestRunner.HTMLTestRunner(
            report_title='Badmusic Test',
            report_name='BadMusic'
        )
    itersuite = unittest.TestLoader().loadTestsFromTestCase(Badmusictest)
    runner.run(itersuite)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Extract MozReview patch data and write out the patch data as a HTML directory.
"""

import argparse
from collections import namedtuple

import os
from time import sleep

import requests

API_BASE_URL = 'https://reviewboard.mozilla.org/api'

PATCHNAME_TEMPLATE = "r{}-diff{}.patch"

REVISION_INDEX_TEMPLATE = """
<!DOCTYPE html>

<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>MozReview patch archive for revision {revision}</title>

MozReview patch archive for revision {revision}

{latest_link}
{difflist_links}

"""

DIFF_LINK_TEMPLATE = """
{text}
"""

DiffData = namedtuple('DiffData', 'review_id diff_id patch')

def call_api(path):
    response = requests.get(API_BASE_URL + path)
    response.raise_for_status()
    return response

def get_diff_count(review_id):
    path = '/review-requests/{}/diffs/'.format(review_id)
    data = call_api(path).json()
    return int(data['total_results'])

def get_patch_for_diff(review_id, diff_id):
    """Fetch a revision diff's contents as a string of bytes."""
    # Fetch the raw diff via the regular site.  The raw diff we want isn't
    # accessible via the Review Board API.
    url = 'https://reviewboard.mozilla.org/r/{}/diff/{}/raw'.format(review_id,
                                                                    diff_id)
    response = requests.get(url)
    response.raise_for_status()
    return response.content

def gather_diff_data(review_id, diff_id):
    patch = get_patch_for_diff(review_id, diff_id)
    return DiffData(review_id, diff_id, patch)

def save_patch(diffdata, output_dir):
    path = filepath_for_record(diffdata, output_dir)
    with open(path, 'wb') as fd:
        fd.write(diffdata.patch)
    print('wrote: {} {} bytes'.format(path, len(diffdata.patch)))

def filepath_for_record(diffdata, output_dir):
    filename = PATCHNAME_TEMPLATE.format(diffdata.review_id, diffdata.diff_id)
    path = os.path.join(
        revision_directory_name(output_dir, diffdata.review_id), filename)
    return path

def revision_directory_name(output_dir, revision_id):
    return os.path.join(output_dir, str(revision_id))

def ensure_revision_directory(output_dir, revision_id):
    dirname = revision_directory_name(output_dir, revision_id)
    if not os.path.isdir(dirname):
        os.mkdir(dirname)

def write_revision_index(output_dir, revision_id, diff_count):
    """Write an index.html for a directory of patches."""
    latest_diff_id = diff_count

    diff_links = []
    for diff_id in range(1, diff_count + 1):
        filename = PATCHNAME_TEMPLATE.format(revision_id, diff_id)
        link_html = DIFF_LINK_TEMPLATE.format(target=filename, text=filename)
        diff_links.append(link_html)

    # We want the list displayed newest to oldest
    diff_links.reverse()
    difflist_links = '\n'.join(diff_links)

    latest_filename = PATCHNAME_TEMPLATE.format(revision_id, latest_diff_id)
    latest_link = DIFF_LINK_TEMPLATE.format(
        target=latest_filename, text='latest.patch')

    index_html = REVISION_INDEX_TEMPLATE.format(
        revision=revision_id,
        latest_link=latest_link,
        difflist_links=difflist_links)

    index_filename = os.path.join(
        revision_directory_name(output_dir, revision_id), 'index.html')
    with open(index_filename, 'w') as index:
        index.write(index_html)
    print('wrote:', index_filename)

def diff_already_downloaded(revision_id, diff_id, output_dir):
    diff_filename = PATCHNAME_TEMPLATE.format(revision_id, diff_id)
    diff_filepath = os.path.join(output_dir, str(revision_id), diff_filename)
    return os.path.exists(diff_filepath)

def record_revision(revision_id, output_dir, rate_limit, skip_existing):
    """Fetch a revision's patches and save them to the output directory."""
    sleep(rate_limit)

    try:
        diff_count = get_diff_count(revision_id)
    except requests.ConnectionError as err:
        print("error: r{} {}".format(revision_id, err))
        return
    except requests.HTTPError as err:
        if err.response.status_code == 404:
            print("skipped: r{} (not found)".format(revision_id))
        else:
            print("error: r{} {}".format(revision_id, err))
        return

    ensure_revision_directory(output_dir, revision_id)

    updates_made = False

    for diff_id in range(1, diff_count + 1):
        if skip_existing and diff_already_downloaded(revision_id, diff_id, output_dir):
            continue
        else:
            updates_made = True

        sleep(rate_limit)

        try:
            diff_data = gather_diff_data(revision_id, diff_id)
        except (requests.HTTPError, requests.ConnectionError) as err:
            print("error: fetching diff r{} {}: {}".format(revision_id,
                                                           diff_id, err))
            continue

        save_patch(diff_data, output_dir)

    if updates_made:
        write_revision_index(output_dir, revision_id, diff_count)
    else:
        print("skipped: r{} (no new diffs)".format(revision_id))

def parse_revision_range_str(rev_range_str):
    """Parse a revision range string into range start and range end integers.

    Range strings have two forms: 'X', which returns (X,X), or 'X..Y',
    which returns (X,Y).
    """
    rev_range_parts = rev_range_str.split('..')
    if len(rev_range_parts) == 2:
        start_rev = int(rev_range_parts[0])
        end_rev = int(rev_range_parts[1])
    else:
        rev = int(rev_range_str)
        start_rev = rev
        end_rev = rev
    return start_rev, end_rev

def ensure_output_directory(output_dir):
    if not os.path.isdir(output_dir):
        os.mkdir(output_dir)

def parse_args():
    parser = argparse.ArgumentParser(
        description='Save MozReview patches to a directory')
    parser.add_argument(
        'revision', help='The revision or revision range to save.  Can be a single revision of the form \'XXXX\' or a range of revisions of the form \'XXXX..YYYY\'')
    parser.add_argument(
        '--skip-existing',
        action='store_true',
        help='Only download new diffs. (default: %(default)s)'
    )
    parser.add_argument(
        '--limit',
        type=float,
        default=1.0,
        help='Rate-limit API requests to one request every LIMIT seconds.  Accepts floating-point numbers like \'0.5\'. (default: %(default)s seconds)'
    )
    parser.add_argument(
        '--output-dir',
        default='site',
        help='Output directory for HTML pages and patches. (default: \'%(default)s\')'
    )
    return parser.parse_args()

def main(opts):
    start_rev, end_rev = parse_revision_range_str(opts.revision)

    ensure_output_directory(opts.output_dir)

    print("Outputting files to directory '{}'".format(opts.output_dir))
    print("Rate-limiting to {} seconds between requests".format(opts.limit))
    print()

    for rev_id in range(start_rev, end_rev + 1):
        record_revision(rev_id, opts.output_dir, opts.limit, opts.skip_existing)

    print()
    print('Done.')

if __name__ == '__main__':
    opts = parse_args()
    main(opts)


import numpy as np
import matplotlib.pyplot as plt
import os

def chi2(exp, obs):
    return np.sum((exp - obs)**2 / exp)

def Sort2Array(x, y):
    temp = np.array([x,y]).T
    #  print(temp)
    temp = temp[temp[:,0].argsort()].T
    return temp[0], temp[1]

k = 0
directory = "./"
chi2Array = np.array([])
timedelArray = np.array([])
for filename in os.listdir(directory):
    if filename.startswith('disp.') and filename.endswith('.dat'):
        #  print(filename)
        timedelBin = filename.replace('disp.out_magshiftbin_', '').replace('.dat', '')
        timedelBin = int(timedelBin)
        timedelArray = np.append(timedelArray, timedelBin*0.1 - 2.5)
        data = np.genfromtxt(filename, delimiter=" ").T
        #  print(data)
        timedel = data[0]
        measDisp = data[1]
        measFit = data[2]
        chi2Array = np.append(chi2Array, chi2(measFit, measDisp))

        plt.figure(num=k, figsize=(10,6))
        plt.plot(timedel, measDisp, label="measured", marker="o", color="black")
        plt.plot(timedel, measFit, label="fitted", color="black")
        plt.legend()
        plt.xlabel("Time delay [days]")
        plt.ylabel("Dispersion")

        ax = plt.gca()
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)

        plt.savefig(filename + ".pdf", bbox_inches="tight")
        plt.close(k)
        k+=1

timedelArray, chi2Array = Sort2Array(timedelArray, chi2Array)
#  print(chi2Array)
#  print(timedelArray)
plt.figure(num=k, figsize=(10,6))
plt.plot(timedelArray, chi2Array, marker="o", color="black")
plt.xlabel("Magnitude difference")
plt.ylabel("$\chi^2$")
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
plt.savefig("chi2.pdf", bbox_inches="tight")
plt.close(k)
k += 1


import os
import json
import numpy as np

# Torch imports
import torch

from pprint import pprint
from common.WavefrontOBJ import *
import igl

def grouped(iterable, n):
    "s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ..."
    return zip(*[iter(iterable)] * n)

#
# Function to save 3D mesh
#
def save_mesh(path, verts, index, faces, normals):
    mverts = verts #np.reshape(verts, (-1, 3))
    mnorms = np.reshape(normals, (-1, 3))

    print('verts.shape: ', mverts.shape)
    mesh = WavefrontOBJ()
    mesh.vertices = mverts
    mesh.normals = mnorms
    mesh.polygons = faces
    save_obj(mesh, path % (index))
    print('Mesh OBJ saved: ', path % (index))

#
# Function to load OBJ files of 3D meshes
#
def load_all_meshes_to_matrixv2(path_list, path_index_list, labels_list , centre_meshes=False, pattern_out=None):
    # Outputs
    vertices        = 0
    centres         = 0
    faces           = {}
    normals         = 0
    masks           = 0

    vertex_count    = 0
    file_count      = 0

    files_list      = path_list.split(" ")
    label_list      = labels_list.split(" ")
    indices_list    = path_index_list.split()

    frame_start     = 0
    frame_end       = 0
    sequenceLength  = 1

    meshList    = []
    colours     = []
    labels      = []
    laplace     = {}

    sequence_index = 0

    for (start, end), file in zip( grouped(indices_list, 2), files_list):
        current_label = label_list[sequence_index]

        for frame in range(int(start), int(end) +1):
            if file.endswith(".obj"):
                mesh_path = (file % frame)
                if os.path.isfile(mesh_path):
                    if current_label not in laplace:
                        v, f = igl.read_triangle_mesh(mesh_path)
                        l = igl.cotmatrix(v, f)
                        laplace[current_label] = []
                    if current_label not in faces:
                        mesh = load_obj(mesh_path)
                        faces[current_label] = mesh.polygons

                    meshList.append(mesh_path)
                    labels.append(current_label)
                    #print("Loaded ", mesh_path)

        sequence_index = sequence_index + 1
    return meshList, colours, labels, faces, laplace

def load_mesh_to_matrix(path_list, vertex_count,Only_vert=False):
    mesh = load_obj(path_list)
    verts = mesh.vertices
    norms = mesh.normals
    faces = mesh.polygons

    vertices = np.zeros([vertex_count])
    normals = np.zeros([vertex_count])
    mask = np.zeros([vertex_count])

    # Reshape and add to array (x,y,z,1,x,y,z,1,x,y,...)
    flaten = lambda l: [item for sublist in l for item in sublist]

    flat_verts = flaten(verts)
    flat_norms = flaten(norms)

    vertices[:len(flat_verts)] = flat_verts
    normals[:len(flat_norms)] = flat_norms
    mask[:len(flat_verts)] = np.ones([len(flat_verts)])
    if Only_vert:
        return vertices
    else :
        return vertices, normals, faces, mask


from discord.ext import commands
import discord

from aiohttp import request
import os
import json

def nextrank(score):
    #Masters+
    if score >= 10000:
        return "RP"
    #Diamond
    elif score >= 7200:
        return "/" + str(((score - 7200) // 700 + 1) * 700 + 7200) + "RP"
    #Platinum
    elif score >= 4800:
        return "/" + str(((score - 4800) // 600 + 1) * 600 + 4800) + "RP"
    #Gold
    elif score >= 2800:
        return "/" + str(((score - 2800) // 500 + 1) * 500 + 2800) + "RP"
    #Silver
    elif score >= 1200:
        return "/" + str(((score - 1200) // 400 + 1) * 400 + 1200) + "RP"
    #Broze
    else:
        return "/" + str(((score // 300 + 1) * 300)) + "RP"

class Apex(commands.Cog):
    def __init__(self, client):
        self.client = client
        with open('data/apex/weapons.txt') as file:
            self.weapon = json.load(file)
        self.color = {
            "Energy": [
                0x636f1d,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/2/2d/Energy_Ammo.svg/revision/latest/scale-to-width-down/256?cb=20190827163648"
            ],
            "Heavy": [
                0x376654,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/5/55/Heavy_Rounds.svg/revision/latest/scale-to-width-down/254?cb=20190827144859"
            ],
            "Unique": [
                0xcb003c,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/7/7f/Supply_Drop_Sniper_Ammo.svg/revision/latest/scale-to-width-down/273?cb=20200819185728"
            ],
            "Light": [
                0xf49b49,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/a/a5/Light_Rounds.svg/revision/latest/scale-to-width-down/254?cb=20190827144754"
            ],
            "Shotgun": [
                0x831600,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/4/42/Shotgun_Shells.svg/revision/latest/scale-to-width-down/254?cb=20190827163750"
            ],
            "Sniper": [
                0x7c80fb,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/3/3e/Sniper_Ammo.svg/revision/latest/scale-to-width-down/256?cb=20200207225134"
            ],
            "Arrow": [
                0xf3d674,
                "https://static.wikia.nocookie.net/apexlegends_gamepedia_en/images/f/fe/Arrows.svg/revision/latest/scale-to-width-down/512?cb=20210430161624"
            ]
        }

    async def getdata(self, ctx, URL):
        async with request(
                "GET", URL
        ) as response:
            data = await response.json(content_type='text/plain')
            return data

    @commands.command(aliases=["alstats", "stat", "stats"])
    async def apex_stats(self, ctx, id, platform='PC'):
        """`stats <id>` gets apex stats from origin id"""
        URL = "https://api.mozambiquehe.re/bridge?version=5&platform={}&player={}&auth={}".format(
            platform, id, os.getenv('APEX')).replace("'", "")
        data = await self.getdata(ctx, URL)
        rank = data['global']['rank']
        legend = data['legends']['selected']

        embed = discord.Embed(title="", color=0xcdcfbb)
        embed.set_author(name=f"{data['global']['name']}'s stats",
                         icon_url=f"{data['global']['rank']['rankImg']}")
        embed.set_thumbnail(url=f"{legend['ImgAssets']['icon']}")
        embed.add_field(name="Rank",
                        value=f"`{rank['rankName']} {rank['rankDiv']}`\
        \n`{rank['rankScore']}{nextrank(rank['rankScore'])}`",
                        inline=False)
        embed.add_field(name="Active Legend",
                        value=f"`{legend['LegendName']}`",
                        inline=False)

        for index in range(len(legend['data'])):
            embed.add_field(name=f"{legend['data'][index]['name'].title()}",
                            value=f"`{legend['data'][index]['value']}`",
                            inline=False)

        embed.set_footer(
            text=
            f"Level {data['global']['level']} ({data['global']['toNextLevelPercent']}%)"
        )
        await ctx.send(embed=embed)

    @commands.command(aliases=['wep', 'weap'])
    async def weapon(self, ctx, weapon=None):
        """`weapon <weapon>` return weapon stats."""
        if not weapon:
            all = ""
            embed = discord.Embed(title="Apex Legends Weapon List",
                                  color=0xcdcfbb)
            for category in self.weapon:
                for name in self.weapon[category]:
                    all += self.weapon[category][name][
                        'displayName'] + f" `{name}`" + "\n"
                embed.add_field(name=f"{category}",
                                value=f"{all}",
                                inline=False)
                all = ""
            await ctx.send(embed=embed)

        else:
            weapon = weapon.capitalize()
            for category in self.weapon:
                if weapon in self.weapon[category]:
                    type = category
                    data = self.weapon[category][weapon]

                    embed = discord.Embed(
                        title=f"{data['displayName']}",
                        color=self.color[data['ammoType']]
                        [0])  #description=f"{data['ammoType']}",)
                    embed.set_author(name=f"{type}",
                                     icon_url=self.color[data['ammoType']][1])
                    embed.set_image(url=f"{data['imgUrl']}")
                    embed.add_field(name="Damage",
                                    value=f"`{data['damage']}`",
                                    inline=False)
                    embed.add_field(name="Magazine Size",
                                    value=f"`{data['magSize']}`",
                                    inline=True)
                    embed.add_field(name="Reload Speed",
                                    value=f"`{data['reload']}`",
                                    inline=True)
                    embed.add_field(name="RPM",
                                    value=f"`{data['RPM']}`",
                                    inline=True)
                    await ctx.send(embed=embed)
                    break

def setup(client):
    client.add_cog(Apex(client))


# -*- coding: utf-8 -*-
"""
可复用工具箱

Ref:
- Python实现：https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa.html
"""

import random
import string

from cryptography.hazmat.primitives.serialization import load_pem_private_key, load_pem_public_key
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives import hashes
from cryptography.exceptions import InvalidSignature

# ----- 随机函数 -----

def gen_random_str(length: int) -> str:
    return ''.join(random.sample(string.ascii_letters + string.digits, length))

# ----- RS256算法 -----

def rs256_sign_with_pem(message: bytes, private_key_pem: bytes):
    """
    RS256签名算法，即SHA256 with RSA算法；私钥文件为pem格式的字节串

    @:param message: 待签名字符串
    @:param private_key_pem: 私钥pem文件路径
    :return:
    """
    private_key: rsa.RSAPrivateKey = load_pem_private_key(private_key_pem.encode('utf-8'))
    return private_key.sign(
        message,
        padding.PSS(
            mdf=padding.MGF1(hashes.SHA256()),
            salt_length=padding.PSS.MAX_LENGTH
        ),
        hashes.SHA256()
    )

def rs256_verify_with_pem(message: bytes, public_key_pem: bytes):
    """
    RS256签名验证算法；公钥文件为pem格式的字节串
    :param message:
    :param public_key_pem:
    :return:
    """
    public_key: rsa.RSAPublicKey = load_pem_public_key(public_key_pem)
    try:
        public_key.sign(
            message,
            padding.PSS(
                mdf=padding.MGF1(hashes.SHA256()),
                salt_length=padding.PSS.MAX_LENGTH
            ),
            hashes.SHA256()
        )
        return True
    except InvalidSignature:
        return False


import csv
import django_rq

from io import StringIO

from django.db.models import Q
from django import forms
from django.conf import settings
from django.template.loader import render_to_string
from django.utils.safestring import mark_safe
from django.urls import reverse

from academy.backoffice.users.forms import BaseStatusTrainingFormSet
from academy.apps.students.models import Student, Training, TrainingStatus, TrainingMaterial
from academy.apps.accounts.models import User, Inbox
from academy.core.email_utils import construct_email_args
from academy.core.fields import TrainingMaterialField
from academy.apps.graduates.models import Graduate, Rating

from post_office import mail
from model_utils import Choices

class ParticipantsRepeatForm(forms.Form):
    batch = forms.ModelChoiceField(
        label='Angkatan', queryset=Training.objects.order_by('batch'), empty_label="Pilih Angkatan",
        help_text="Pemberitahuan akan dikirim kesemua peserta tidak " \
        "lulus pada angkatan yang dipilih"
    )

    def __init__(self, *args, **kwargs):
        self.users_repeat = []
        super().__init__(*args, **kwargs)

    def clean(self):
        batch = self.cleaned_data['batch']
        students = Student.objects.filter(status=Student.STATUS.participants, training=batch)
        if not students:
            raise forms.ValidationError("Maaf tidak ada peserta pada angkatan ini")

        user_ids = students.values_list("user_id", flat=True)
        users = User.objects.filter(id__in=user_ids)

        for user in users:
            status = user.get_count_training_status()
            if status['graduate'] < settings.INDICATOR_GRADUATED and status['not_yet'] == 0 \
                    and status['repeat'] >= settings.INDICATOR_REPEATED:
                self.users_repeat.append({'user': user, 'status': status})

        if not self.users_repeat:
            raise forms.ValidationError("Tidak ada peserta yang mengulang")

        return batch

    def send_notification(self):
        # get setting appearance
        from academy.apps.offices import utils
        sett = utils.get_settings(serializer=True)

        title = "Maaf, Anda belum bisa lanjut"

        context = {
            'host': settings.HOST,
            'email_title': title,
            'indicator': settings.INDICATOR_GRADUATED
        }
        context.update(sett)

        for data in self.users_repeat:
            context['user'] = data['user']
            context['graduate'] = data['status']['graduate']
            html_message = render_to_string('emails/participants_repeat.html', context=context)

            Inbox.objects.create(user=data['user'], subject=title, content=html_message)

            kwargs = construct_email_args(
                recipients=data['user'].email,
                subject=title,
                content=html_message
            )
            student = data['user'].get_student()
            student.status = Student.STATUS.repeat
            student.save(update_fields=['status'])
            django_rq.enqueue(mail.send, **kwargs)

        return len(self.users_repeat)

class AddTrainingStatus(forms.ModelForm):
    training_material = TrainingMaterialField()
    STATUS = Choices(
        (1, 'not_yet', 'Belum'),
        (2, 'graduate', 'Lulus'),
        (3, 'repeat', 'Ulang'),
    )
    status = forms.ChoiceField(choices=STATUS, required=False, label="")

    def __init__(self, user, *args, **kwargs):
        self.user = user
        super().__init__(*args, **kwargs)

    class Meta:
        model = TrainingStatus
        fields = ('training_material', 'status')

    def clean(self):
        cleaned_data = super().clean()
        if TrainingStatus.objects.filter(user=self.user, training_material=cleaned_data['training_material']).exists():
            raise forms.ValidationError(
                f"{self.user.name} sudah memiliki materi {cleaned_data['training_material']}"
            )
        return cleaned_data

    def save(self, *args, **kwargs):
        training_status = super().save(commit=False)
        training_status.user = self.user
        training_status.save()
        return training_status

class GraduateTrainingStatusFormSet(BaseStatusTrainingFormSet):

    def __init__(self, graduate, *args, **kwargs):
        """
        create label any button to action delete training status
        training material data get form user, not from batch
        """
        self.graduate = graduate
        self.user = graduate.user
        self.trainig = {
            'materials': [],
            'status': []
        }
        for ts in self.user.training_status.all():
            if ts.training_material:
                self.trainig['status'].append(ts)
                self.trainig['materials'].append(ts.training_material)

        super().__init__(training_materials=self.trainig['materials'], *args, **kwargs)
        for i, form in enumerate(self.forms):
            # skip or continue last loop
            if i > len(self.training_materials) - 1:
                continue
            material = self.trainig['materials'][i]
            status = self.trainig['status'][i]

            # create button to combine with label
            url = reverse('backoffice:graduates:delete_training_status', args=[status.id, graduate.id])
            html_button = ''
            label = mark_safe(f"{html_button} {material.code} - {material.title}")

            form.fields['training_material'].label = label

class BaseFilterForm(forms.Form):
    name = forms.CharField(
        widget=forms.TextInput(attrs={'placeholder': 'Cari: nama/username'}),
        required=False
    )
    batch = forms.ModelChoiceField(
        queryset=Training.objects.order_by('batch'), empty_label="Pilih Angkatan", required=False
    )
    STATUS = (
        (None, 'Pilih Status'),
        (True, 'Tersalurkan'),
        (False, 'Belum Tersalurkan'),
    )
    is_channeled = forms.ChoiceField(label='Tersalurkan', required=False, choices=STATUS)

    def __init__(self, *args, **kwargs):
        self.graduates = None
        super().__init__(*args, **kwargs)

    def get_data(self):
        name = self.cleaned_data['name']
        batch = self.cleaned_data['batch']
        is_channeled = self.cleaned_data['is_channeled']
        graduates = Graduate.objects.select_related('user')

        if batch:
            graduates = graduates.filter(student__training=batch)
        if is_channeled:
            graduates = graduates.filter(is_channeled=is_channeled)
        if name:
            graduates = graduates.filter(
                Q(user__first_name__icontains=name) | Q(user__last_name__icontains=name) |
                Q(user__username__icontains=name)
            )

        self.graduates = graduates
        return self.graduates

    def generate_to_csv(self):
        csv_buffer = StringIO()
        writer = csv.writer(csv_buffer)
        writer.writerow([
            'No.', 'Nama', 'No.Sertifikat', 'Email', 'No. Ponsel', 'Tersalurkan'
        ])
        for index, graduate in enumerate(self.graduates, 1):
            writer.writerow([
                index, graduate.user.name, graduate.certificate_number,
                graduate.user.email, graduate.user.phone,
                'Ya' if graduate.is_channeled else 'Tidak'
            ])

        return csv_buffer

class GraduateHasChanneledForm(forms.Form):
    channeled_at = forms.CharField(label='Tersalurkan pada', max_length=150, help_text='Nama Perusahaan')
    graduate_id = forms.CharField(max_length=100, label='', widget=forms.HiddenInput())

    def save(self):
        cleaned_data = super().clean()
        graduate = Graduate.objects.get(id=cleaned_data['graduate_id'])
        graduate.channeled_at = cleaned_data['channeled_at']
        graduate.is_channeled = True
        graduate.save()

class RatingForm(forms.ModelForm):

    class Meta:
        model = Rating
        exclude = ('graduate',)
        help_texts = {
            'respondent_name': 'Nama reponden bisa perusahaan, personal, atau jabatan.'
        }


import os
import sys

import numpy as np
import pandas as pd
from scipy.stats import pearsonr, spearmanr

from etspredict.prepare_data.prepare_connectomes import load_timeseries
from etspredict.selective_connectomes import ets_amplitudes
from etspredict.timeseries import get_edge_timeseries

def get_subject_rss(subject, session, parcellation, preproc="GS"):
    timeseries = load_timeseries(
        "hcp", subject, parcellation, session, preproc
    )
    ets = get_edge_timeseries(timeseries)

    return ets_amplitudes(ets)

def get_subject_confound(subject, session, fieldname):

    aml_dir = os.path.join(
        "/data", "group", "appliedml", "data", "HCP1200_Confounds_tsv_withFD"
    )
    confounds_file = os.path.join(
        f"{subject}",
        "MNINonLinear",
        "Results",
        f"rfMRI_{session}",
        f"Confounds_{subject}.tsv",
    )
    fd_file = os.path.join(aml_dir, confounds_file)

    return np.array(pd.read_csv(fd_file, sep="\t")[fieldname])

def main(subject, session, parcellation, correlation_method, preproc="GS"):

    fd = get_subject_confound(subject, session, fieldname="FD")
    rss = get_subject_rss(subject, session, parcellation, preproc=preproc)
    gs = get_subject_confound(subject, session, fieldname="GS")

    if correlation_method in ["pearson"]:
        correlation_fd, _ = pearsonr(rss, fd)
        correlation_gs, _ = pearsonr(rss, gs)
    elif correlation_method in ["spearman"]:
        correlation_fd, _ = spearmanr(rss, fd)
        correlation_gs, _ = spearmanr(rss, gs)
    else:
        raise NotImplementedError(f"{correlation_method} not implemented")

    print("correlation done")
    with open(
        f"../results/fd_rss_correlations/"
        f"{subject}_{session}_{parcellation}"
        f"_{preproc}_{correlation_method}.txt",
        "w",
    ) as f:
        f.write(f"{correlation_fd}")

    with open(
        f"../results/gs_rss_correlations/"
        f"{subject}_{session}_{parcellation}"
        f"_{preproc}_{correlation_method}.txt",
        "w",
    ) as f:
        f.write(f"{correlation_gs}")
    print("files saved")

if __name__ == "__main__":

    main(
        subject=sys.argv[1],
        session=sys.argv[2],
        parcellation=sys.argv[3],
        correlation_method=sys.argv[4],
        preproc="GS",
    )

    main(
        subject=sys.argv[1],
        session=sys.argv[2],
        parcellation=sys.argv[3],
        correlation_method=sys.argv[4],
        preproc="no_GS",
    )


from django.contrib import admin
from django.utils.translation import ugettext_lazy as _

from .models import ServiceCategory, Service

class ServiceCategoryAdmin(admin.ModelAdmin):
	pass

class ServiceAdmin(admin.ModelAdmin):
    list_display = (
        'name', 'category',
        'price',
        'due_period', 'reminder_day_count_before_due_date', 'reminder_day_count_after_due_date',
    )
    list_filter = ('category',)
    search_fields = ('name',)
    fieldsets = (
        (_('Basic Profile'), {'fields': ('category', 'name',)}),
        (_('Price Fields'), {'fields': ('price',)}),
        (_('Reminder Fields'),
        	{'fields': ('due_period', 'reminder_day_count_before_due_date', 'reminder_day_count_after_due_date',)}),
    )

admin.site.register(ServiceCategory, ServiceCategoryAdmin)
admin.site.register(Service, ServiceAdmin)


import pytest
import subprocess
from deployer.env.go import GoEnvironment as Env

@pytest.fixture(scope="module")
def bootstrap():
    # make settable by env var
    current_env = subprocess.check_output("juju env", shell=True).strip()
    # make switchable by env var
    subprocess.check_call('juju bootstrap --upload-tools')
    return current_env

@pytest.yield_fixture(scope="module")
def juju_environment(bootstrap):

    current_env = subprocess.check_output("juju env", shell=True).strip()
    env = Env(current_env, None)
    yield Env
    env.reset(terminate_machines=True)


from django.urls import path
from .views import ping, PackageAPI, PackageInfoAPI, TopicAPI, QuestionCreationAPI, QuestionInfoAPI, SubmitScoreAPI, \
    PackageScoreBoard

urlpatterns = [
    path('ping', ping),
    path('packages', PackageAPI.as_view()),
    path('packages/<int:pk>', PackageInfoAPI.as_view()),
    path('packages/<int:pk>/scoreboard', PackageScoreBoard.as_view()),
    path('packages/<int:pk>/submit', SubmitScoreAPI.as_view()),
    path('topics', TopicAPI.as_view()),
    path('questions', QuestionCreationAPI.as_view()),
    path('questions/<int:pk>', QuestionInfoAPI.as_view()),
]


from movements import app
from movements.forms import MovementForm
from flask import Flask, render_template, request, url_for, redirect
import sqlite3
from datetime import date, datetime
import time
import requests #from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
import json
import numpy as np
import matplotlib.pyplot as plt

API_KEY =  app.config['API_KEY']
DBFILE = app.config['DBFILE']

def consulta(query, params=()):
    ## función 1: establecer conexión con sqlite3
    conn = sqlite3.connect(DBFILE)
    c = conn.cursor()
    c.execute(query, params)
    conn.commit()
    filas = c.fetchall()
    conn.close

    ## función 2: dar formato a la consulta como lista de diccionarios
    if len(filas) == 0:
        return filas

    columnNames = []
    for columnName in c.description:
        columnNames.append(columnName[0])

    listaDeDiccionarios = []

    for fila in filas:
        d = {}
        for ix, columnName in enumerate(columnNames): # enumerate() te da una tupla con el índice y el elemento.
            d[columnName] = fila[ix]
        listaDeDiccionarios.append(d)

    return listaDeDiccionarios

def peticionAPI(specific_url):
    response = requests.get(specific_url)
    if response.status_code == 200:
        api = response.json()
        return api
    else: #(ConnectionError, Timeout, TooManyRedirects) as e:
        raise Exception("Problema de consulta tipo {}".format(response.status_code))

def calc_monedero():
    lista_monedas = ['BTC','ADA','BCH','BSV','EOS','ETH','LTC','BNB','TRX','USDT','XLM','XRP']
    monedero = {} # cuantas monedas tengo de cada cripto. No incluye EUR.
    for moneda in lista_monedas:
        query = "SELECT sum(case when to_currency = ? then to_quantity else 0 end) - sum(case when  from_currency = ? then from_quantity else 0 end) as saldo_moneda FROM movements;"
        tmp = consulta(query, (moneda, moneda))
        saldo = tmp[0]['saldo_moneda']
        monedero[moneda] = saldo
    return monedero

def validarConversion(amount, simbolo, convert):
    monedero = calc_monedero()
    error = None
    try:
        if float(amount) <= 0:
            error = 'Cifra no válida. Entre cantidad superior a 0.'
            return error
    except:
        error = 'Cifra no válida. Use el punto para separar decimales. No se admiten otros símbolos.'
        return error

    if simbolo != 'EUR' and monedero[simbolo] < float(amount):
        error = 'Sólo dispone de {:.4f} {} para gastar. Consulte el monedero para ver la cantidad real.'.format(monedero[simbolo], simbolo)
        return error

    if simbolo == convert:
        error = 'Las monedas From y To deben ser diferentes.'
        return error

    if simbolo == 'EUR' and convert != 'BTC':
        error = 'Con EUR sólo puedes comprar BTC.'
        return error

    if simbolo != 'BTC' and convert == 'EUR':
        error = 'Sólo puede comprar EUR con BTC.'
        return error

@app.route('/')
def listaMovimientos(): # mostrar tabla SQL
    query = "SELECT date, time, from_currency, from_quantity, to_currency, to_quantity, from_quantity/to_quantity as precio_unitario_to FROM movements;"
    mensaje = ""
    try:
        compras = consulta(query)
    except Exception as e:
        print("**ERROR**: Acceso base de datos - {}".format(type(e).__name__))
        mensaje = "Error en acceso a la base de datos. Consulte con el administrador."
        return render_template('movementsList.html', error=mensaje)

    monedero_actual = calc_monedero()
    return render_template('movementsList.html', datos=compras, monedero=monedero_actual, error=mensaje)

@app.route('/purchase', methods=['GET', 'POST'])
def purchase():
    # 1) Formulario vacío:
    if request.method == 'GET':
        form_vacio = MovementForm()
        try:
            monedero = calc_monedero()
        except Exception as e:
            print("**ERROR**: Acceso base de datos - {}".format(type(e).__name__))
            mensaje = "Error en acceso a la base de datos. Consulte con el administrador."
            return render_template('movementsList.html', error=mensaje)

        lista_from = ['EUR'] # crear una lista de monedas disponibles para FROM: EUR + las que tengan cantidad > 0 en mi monedero

        try:
            for moneda, q in monedero.items():
                if q>0:
                    lista_from.append(moneda)
            form_vacio.from_currency.choices = lista_from # choices necesita una lista
        except:
            monedero = 'vacio'
            form_vacio.from_currency.choices = lista_from

        return render_template('purchase.html', form=form_vacio, vacio='yes', monedero=monedero)

    else: #2) request.method == 'POST': Grabar o Calcular
        form = MovementForm()
        form.from_currency.choices = [request.form.get('from_currency')] # bloqueamos la lista FROM con  la moneda elegida
        form.to_currency.choices = [request.form.get('to_currency')] # bloqueamos la lista TO con  la moneda elegida
        monedero = calc_monedero()

        if  request.form.get('submit') == 'Grabar' and form.validate():
            try:
                query = "INSERT INTO movements (date, time, from_currency, from_quantity, to_currency, to_quantity) VALUES (?,?,?,?,?,?);"
                consulta(query, (request.form.get("date"), request.form.get("time"), request.form.get("from_currency"),
                                request.form.get("from_quantity"), request.form.get("to_currency"), request.form.get("to_quantity")))

                return redirect(url_for('listaMovimientos'))
            except:
                error = "Error: Debe efectuar el cálculo (botón calculadora) antes de validar la compra."
                return render_template('purchase.html', form=form, vacio='yes', error=error, monedero=monedero)
        else:
            amount = request.form.get('from_quantity') # x "simbolo" convierte a "convert"
            simbolo = request.form.get('from_currency')
            convert = request.form.get('to_currency')

            #3) Errores en la conversión de monedas:
            error = validarConversion(amount, simbolo, convert)
            if error != None:
                return render_template('purchase.html', form=form, vacio='yes', error=error, monedero=monedero)

            # 4) Finalmente: consulta a la API
            try:
                url = 'https://pro-api.coinmarketcap.com/v1/tools/price-conversion?amount={}&symbol={}&convert={}&CMC_PRO_API_KEY={}'.format(amount, simbolo, convert, API_KEY)
                dicc = peticionAPI(url)

                q_to = dicc['data']['quote'][convert]['price'] # cantidad de monedas que podemos comprar (viene del API)
                precioUnitario = float(amount)/q_to
                fecha_compra = time.strftime("%d/%m/%Y") # fecha y hora del momento de la consulta API
                hora_compra = time.strftime("%X")

                return render_template('purchase.html', vacio='no', form=form, q_to=q_to, precioUnitario=precioUnitario, hora_compra=hora_compra, fecha_compra=fecha_compra, monedero=monedero, amount=amount)
            except Exception as error:
                print("**ERROR**: Conexion url en PURCHASE - {}".format(type(error).__name__), error) # Failed to establish a new connection
                error = "Problema de conexión. Contacte con el administrador."
                return render_template('purchase.html', vacio='yes', form=form, error=error, monedero=monedero)

@app.route('/status')
def status():
    # 1)
    query1 = "SELECT SUM(from_quantity) as eur_from FROM movements WHERE from_currency='EUR';"
    try:
        total_diccionario = consulta(query1)
    except Exception as e:
        print("**ERROR**: Acceso base de datos - {}".format(type(e).__name__))
        mensaje = "Error en acceso a la base de datos. Consulte con el administrador."
        return render_template('movementsList.html', error=mensaje)

    total_eur_invertido = total_diccionario[0]['eur_from']

    #2)
    query2 = 'SELECT sum(case when to_currency = "EUR" then to_quantity else 0 end) - sum(case when  from_currency = "EUR" then from_quantity else 0 end) as saldo_eur from movements;'
    eur_diccionario = consulta(query2)
    saldo_euros_invertidos = eur_diccionario[0]['saldo_eur']

    #3) obtener precio unitario en EUR de todas las criptomonedas:
    simbolo = "BTC,ETH,XRP,LTC,BCH,BNB,USDT,EOS,BSV,XLM,ADA,TRX" # no debe tener espacios en blanco!
    convert = 'EUR'

    url = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?symbol={}&convert={}&CMC_PRO_API_KEY={}'.format(simbolo, convert, API_KEY)
    try:
        dicc = peticionAPI(url)
    except Exception as error:
        print("**ERROR**: Conexion url en STATUS - {}".format(type(error).__name__), error)
        error = "Problemas de conexión. Contacte con el administrador."
        return render_template('status.html', invertido=0 , actual=0, error_conexion = error)

    lista_monedas = ['BTC', 'ETH', 'XRP', 'LTC', 'BCH','BNB', 'USDT','EOS','BSV','XLM','ADA','TRX']
    pu_crypto_eur = {}
    id_cryptos = {}
    for moneda in lista_monedas: # crear diccionario pu_crypto_eur
        precio = dicc['data'][moneda]['quote']['EUR']['price']
        id_crypto = dicc['data'][moneda]['id']
        pu_crypto_eur[moneda] = precio
        id_cryptos[moneda] = id_crypto #print(id_cryptos)

    #4)
    monedero = {} # cuantas monedas tengo de cada cripto
    monedero_eur = [] # valor en EUR de las monedas que tengo

    for moneda in lista_monedas:
        query3 = "SELECT sum(case when to_currency = ? then to_quantity else 0 end) - sum(case when  from_currency = ? then from_quantity else 0 end) as saldo_moneda FROM movements;"
        tmp = consulta(query3, (moneda, moneda))
        saldo = tmp[0]['saldo_moneda']

        monedero[moneda] = saldo

        precio_unitario = float(pu_crypto_eur[moneda])
        try:
            monedero_eur.append(saldo * precio_unitario)
        except:
            return render_template('status.html', invertido=0 , actual=0)

    valor_total_cryptos = 0
    for item in monedero_eur:
        valor_total_cryptos += item

    valor_actual = total_eur_invertido + saldo_euros_invertidos + valor_total_cryptos # todo en euros!

    return render_template('status.html', invertido=total_eur_invertido , actual=valor_actual)


import pandas as pd
import numpy as np
import os
import nltk
import sys
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from scipy import sparse

import texthero as hero
from texthero import preprocessing
from nltk import FreqDist
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn import metrics
import nltk
# nltk.download('punkt')
# nltk.download('averaged_perceptron_tagger')
from collections import Counter

from mpl_toolkits.mplot3d import Axes3D
import math

pd.set_option('mode.chained_assignment', None)

np.set_printoptions(threshold=10)

F = [0.075, 0.2, 0.5,0.75, 1]
# F = [0.075]

profiles = pd.read_csv('profiles.csv',sep=',',nrows = 10000)
df = profiles
df.loc[df.sex == 'm', 'sex'] = 1
df.loc[df.sex == 'f', 'sex'] = 0

#clean essays (how?)
essays=['essay0','essay1','essay2','essay3','essay4','essay5','essay6','essay7','essay8','essay9']
# df = profiles[essays]
for i in essays:
    profiles[i] = hero.clean(profiles[i])
    profiles[i] = [n.replace('br','') for n in profiles[i]]

Test_ac = []
Test_er = []
Train_ac = []
Train_er = []

#different fraction of data
for t_frac , K in zip(F, [[50], [200], [200], [500], [500]]) :
    print('t_frac',t_frac)
    temp = profiles.sample(frac = t_frac,  random_state = 32)
    pf = temp.copy()

    test_ac = []
    train_ac = []

    #split training and testing
    S = np.array_split(pf, 10)
    for ite in range(10):
        print('ite', ite)

        testSet = S[ite]
        temp = pf.drop(testSet.index)
        trainingSet = temp.copy()

        #---------work on training data
        df = trainingSet[essays]
        #combine 9 essays of every person
        df['dumb'] = np.zeros(len(df))
        df['row essays'] = df['essay0'] + " " + df['essay1'] + " " + df['essay2'] + " " + df['essay3'] + " " + df['essay4'] + " " + df['essay5'] + " " + df['essay6'] + " " + df['essay7'] + " " + df['essay8'] + " " + df['essay9']
        df = df.drop(columns = essays)

        df_essays_per_person = df.drop(columns = ['dumb'])
        df_essays_per_person.to_csv('essays_per_person.csv', sep=',',  index=False)

        #combine all essays
        df_essay_all_persons = df.groupby('dumb')['row essays'].apply(' '.join).reset_index()
        df_essay_all_persons = df_essay_all_persons.drop(columns = ['dumb'])
        df_essay_all_persons.to_csv('essay_all_persons.csv', sep=',',  index=False)

        #build vocabulary array of df_essay_all_persons
        string = df_essay_all_persons.iloc[0]['row essays']

        tokens = nltk.word_tokenize(string)
        tags = nltk.pos_tag(tokens)
        voc_pos_dict = Counter( tag for word,  tag in tags)
        voc_list = list(voc_pos_dict.keys())

        #build vocabulary frequency dictionaries of each person
        dict_traing_each_person = []
        for i in range(len(df)):
            # print(i)
            string = df.iloc[i]['row essays']
            tokens = nltk.word_tokenize(string)
            tags = nltk.pos_tag(tokens)
            dict_ = Counter( tag for word,  tag in tags)
            dict_traing_each_person.append(dict_)

        #convert training data essays into word frequencies
        X_train = np.zeros((len(df), len(voc_list)))
        for i in range(len(df)):
            # print('0', i)
            for j in range(len(voc_list)):
                if voc_list[j] in dict_traing_each_person[i]:
                    X_train[i][j] = dict_traing_each_person[i][voc_list[j]]

        Y_train = trainingSet.loc[:,['sex']].values
        Y_train = Y_train.astype(int)
        Y_train = Y_train.T
        Y_train = Y_train.flatten()

        #---------work on testSet
        df = testSet[essays]
        #combine 9 essays of every person
        df['dumb'] = np.zeros(len(df))
        df['row essays'] = df['essay0'] + " " + df['essay1'] + " " + df['essay2'] + " " + df['essay3'] + " " + df['essay4'] + " " + df['essay5'] + " " + df['essay6'] + " " + df['essay7'] + " " + df['essay8'] + " " + df['essay9']
        df = df.drop(columns = essays)

        df_essays_per_person = df.drop(columns = ['dumb'])
        df_essays_per_person.to_csv('essays_per_person.csv', sep=',',  index=False)

        #build vocabulary frequency dictionaries of each person
        dict_test_each_person = []
        for i in range(len(df)):
            # print('2',i)
            string = df.iloc[i]['row essays']
            tokens = nltk.word_tokenize(string)
            tags = nltk.pos_tag(tokens)
            dict_ = Counter( tag for word,  tag in tags)
            dict_test_each_person.append(dict_)

        #convert test data essays into word frequencies
        X_test = np.zeros((len(df), len(voc_list)))
        for i in range(len(df)):
            # print('3',i)
            for j in range(len(voc_list)):
                if voc_list[j] in dict_test_each_person[i]:
                    X_test[i][j] = dict_test_each_person[i][voc_list[j]]

        Y_test = testSet.loc[:,['sex']].values
        Y_test = Y_test.astype(int)
        Y_test = Y_test.T
        Y_test = Y_test.flatten()

        #----------PCA
        # Standardizing the features
        X_train = StandardScaler().fit_transform(X_train)
        # np.savetxt("X_train.csv", X_train, delimiter = ",")
        X_test = StandardScaler().fit_transform(X_test)

        pca = PCA(.95)
        pca.fit(X_train)

        X_train = pca.transform(X_train)
        X_test = pca.transform(X_test)

        aa =  pca.explained_variance_ratio_

        # #plot first three Principle components
        # x = X_train[:, 0]
        # y = X_train[:, 1]
        # z = X_train[:, 2]

        # group = Y_train
        # cdict = {0: 'red', 1: 'blue'}

        # fig = plt.figure()
        # ax = Axes3D(fig)
        # for g in np.unique(group):
        #     ix = np.where(group == g)
        #     ax.scatter(x[ix], y[ix], z[ix], c = cdict[g], label = g, s = 30, alpha=0.5)

        # ax.set_zlabel('Third Principle Component', fontdict={'size': 15})
        # ax.set_ylabel('Second Principle Component', fontdict={'size': 15})
        # ax.set_xlabel('First Principle Component', fontdict={'size': 15})

        # ax.legend()
        # plt.show()

        #------------kmeans

        sil_score_max = -1 #this is the minimum possible score

        distortions = []

        for i in K:

            # print(i)
            km = KMeans(
                n_clusters=i, init='random',  max_iter=300,

            ).fit(X_train)

            Cluster_train = km.labels_
            Cluster_test = km.predict(X_test)
            distortions.append(km.inertia_)
            sil_score = silhouette_score(X_train, Cluster_train)
            # print("s SCORE",i ,sil_score)
            k = i

        # print('sex_cluster',cluster_sex)

        # # plot
        # fig3, ax3 = plt.subplots()
        # plt.plot(K, distortions, marker='o')
        # plt.title('Fraction = ' + str(t_frac))
        # plt.xlabel('Number of clusters')
        # plt.ylabel('WC-SSD')
        # plt.show()

        #assign gender to each cluster
        cluster_sex_train = np.array([Cluster_train, Y_train])
        cluster_sex_train = cluster_sex_train.T

        cluster_sex_map = []
        for i in range(k):
            cluster = cluster_sex_train[cluster_sex_train[:,0] == i, :]
            sex = cluster[:,1]
            sex = np.mean(sex)
            # cluster_sex_map.append(sex)
            if sex >= 0.5:
                cluster_sex_map.append(1)
            else:
                cluster_sex_map.append(0)

        #look at testset and calculate sex prediction accuracy

        n_match = 0

        # for i in range(len(Cluster_test)):

        #     p = cluster_sex_map[Cluster_test[i]]
        #     rand_sex = np.random.binomial(1, p, 1)

        #     if rand_sex[0] == Y_test[i]:
        #         n_match += 1

        for i in range(len(Cluster_test)):
            if cluster_sex_map[Cluster_test[i]] == Y_test[i]:
                n_match += 1

        test_acc = n_match/len(Cluster_test)
        test_ac.append(test_acc)
        # print(test_acc)

        #look at training and calculate sex prediction accuracy
        n_match = 0
        for i in range(len(Cluster_train)):
            if cluster_sex_map[Cluster_train[i]] == Y_train[i]:
                n_match += 1

        train_acc = n_match/len(Cluster_train)
        train_ac.append(train_acc)
        # print(train_acc)

    #calculate mean of test and train acc and error after 10 fold CV
    Test_ac.append(np.mean(test_ac))
    Test_er.append(np.std(test_ac)/math.sqrt(10))
    Train_ac.append(np.mean(train_ac))
    Train_er.append(np.std(train_ac)/math.sqrt(10))

    print('test_ac',test_ac)
    print('train_ac',train_ac)

#plot acc over number of samples
fig2, ax2 = plt.subplots()

plt.xlabel('Training Examples')
plt.ylabel('Model Accuracy')

n_samples = [element * 9000 for element in F]

plt.plot(n_samples,Test_ac,label="Test Accuracy")
plt.errorbar(n_samples,Test_ac,yerr=Test_er,fmt='o',ecolor='r',color='b',elinewidth=2,capsize=4)
print('Train_er',Train_er)
print('Test_er',Test_er)

plt.plot(n_samples,Train_ac,label="Training Accuracy")
plt.errorbar(n_samples,Train_ac,yerr=Train_er,fmt='o',ecolor='r',color='b',elinewidth=2,capsize=4)

plt.legend(loc=[0.5, 1], prop={'size': 12})

Machine learning/Predicting gender using Tinder profile/colab_notebooks-20201217T063810Z-001/Kmeans.py


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import warnings

# Credits to the contributors at fast.ai
def dropout_mask(x, sz, p):
    return x.new(*sz).bernoulli_(1-p).div_(1-p)

class RNNDropout(nn.Module):
    def __init__(self, p=0.5):
        super().__init__()
        self.p=p

    def forward(self, x):
        if not self.training or self.p == 0.:
            return x
        m = dropout_mask(x.data, (x.size(0), 1, x.size(2)), self.p)
        return x * m

class EmbeddingDropout(nn.Module):
    def __init__(self, emb, embed_p):
        super(EmbeddingDropout, self).__init__()
        self.emb = emb
        self.embed_p = embed_p
        self.pad_idx = self.emb.padding_idx
        if self.pad_idx is None: self.pad_idx = -1

    def forward(self, words, scale=None):
        if self.training and self.embed_p != 0:
            size = (self.emb.weight.size(0),1)
            mask = dropout_mask(self.emb.weight.data, size, self.embed_p)
            masked_embed = (self.emb.weight * mask)
        else:
            masked_embed = (self.emb.weight)
        if scale:
            masked_embed.mul_(scale)
        out = F.embedding(words, masked_embed, self.pad_idx, self.emb.max_norm,
                          self.emb.norm_type, self.emb.scale_grad_by_freq, self.emb.sparse)
        return out

class WeightDropout(nn.Module):
    def __init__(self, module, weight_p, layer_names=['weight_hh_l0']):
        super(WeightDropout, self).__init__()
        self.module,self.weight_p,self.layer_names = module,weight_p,layer_names
        for layer in self.layer_names:
            #Makes a copy of the weights of the selected layers.
            w = getattr(self.module, layer)
            self.register_parameter(f'{layer}_raw', nn.Parameter(w.data))
            self.module._parameters[layer] = F.dropout(w, p=self.weight_p, training=False)

    def _setweights(self):
        "Apply dropout to the raw weights."
        for layer in self.layer_names:
            raw_w = getattr(self, f'{layer}_raw')
            self.module._parameters[layer] = F.dropout(raw_w, p=self.weight_p, training=self.training)

    def forward(self, *args):
        self._setweights()
        with warnings.catch_warnings():
            #To avoid the warning that comes because the weights aren't flattened.
            warnings.simplefilter("ignore")
            return self.module.forward(*args)

    def reset(self):
        for layer in self.layer_names:
            raw_w = getattr(self, f'{layer}_raw')
            self.module._parameters[layer] = F.dropout(raw_w, p=self.weight_p, training=False)
        if hasattr(self.module, 'reset'): self.module.reset()


from django.core.urlresolvers import reverse
from django.http.response import HttpResponseRedirect
from django.utils.functional import wraps
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from cyano.models import Entry

import cyano.models as models
from cyano.helpers import render_queryset_to_response_error, getModel

def __assign_if_not_false(kw, key, obj):
    if obj:
        kw[key] = obj

def resolve_to_objects(function):
    @wraps(function)
    def wrapper(request, *args, **kw):
        # False instead of None, None is default value for passed but not set
        species_wid = kw.pop("species_wid", False)
        model_type = kw.pop("model_type", False)
        wid = kw.pop("wid", False)

        species = False
        model = False
        item = False

        if species_wid:
            species = models.Species.objects.for_wid(species_wid, get=False)
            try:
                species = species.get()
            except ObjectDoesNotExist as e:
                return render_queryset_to_response_error(
                    request,
                    error=404,
                    msg="The requested species \"{}\" was not found.".format(species_wid),
                    msg_debug=repr(e))
            except MultipleObjectsReturned as e:
                return render_queryset_to_response_error(
                    request,
                    error=500,
                    msg="Database error for species \"{}\". Please report this to an administrator!.".format(
                        species_wid),
                    msg_debug=repr(e))
        if model_type:
            model = getModel(model_type)
            # issubclass test needed to allow /add/Species/ but not /species_wid/Species/
            if model is None or (species_wid and not issubclass(model, models.SpeciesComponent)):
                return render_queryset_to_response_error(
                    request,
                    species=species,
                    error=404,
                    msg="The requested species \"{}\" has no model \"{}\".".format(species_wid, model_type))
        if model and wid:
            item = model.objects.for_species(species).for_wid(wid, get=False)
            try:
                item = item.get()
            except ObjectDoesNotExist as e:
                return render_queryset_to_response_error(
                    request,
                    error=404,
                    species=species,
                    model=model,
                    msg="The requested species \"{}\" has no item \"{}\" of type \"{}\".".format(species_wid, wid,
                                                                                model_type),
                    msg_debug=repr(e))
            except MultipleObjectsReturned as e:
                return render_queryset_to_response_error(
                    request,
                    error=500,
                    species=species,
                    model=model,
                    msg="Database error for species \"{}\" accessing item \"{}\" of type \"{}\"."
                        " Please report this to an administrator!".format(
                        species_wid, wid, model_type),
                    msg_debug=repr(e))

        # Prepare keyword arguments
        __assign_if_not_false(kw, "species", species)
        __assign_if_not_false(kw, "model", model)
        __assign_if_not_false(kw, "item", item)

        return function(request, *args, **kw)

    return wrapper

def global_permission_required(permission):
    """Checks whether a user has a permission for the specified entry
    """

    def decorator(function):
        @wraps(function)
        def wrapper(request, *args, **kw):
            from cyano.views import login

            user = request.user

            if not user.is_authenticated():
                # Special handling for guests
                user = User.objects.get(username="guest")

            # Admins always have full access
            if user.is_superuser:
                return function(request, *args, **kw)

            # Global perm check
            if user.profile.has_perm(permission):
                return function(request, *args, **kw)

            # No perm fallthrough
            msg = "You need global permission \"{}\"".format(permission)

            return login(
                request,
                species=None,
                error=403,
                message=msg,
                force_next=request.build_absolute_uri(),
                required_perm=permission)

        return wrapper

    return decorator

def permission_required(permission):
    """Checks whether a user has a permission for the specified entry
    """

    def decorator(function):
        @wraps(function)
        def wrapper(request, *args, **kw):
            from cyano.views import login

            species = kw.get("species", False)
            item = kw.get("item", False)

            user = request.user

            if not user.is_authenticated():
                # Special handling for guests
                user = User.objects.get(username="guest")
            else:
                pass

            # Admins always have full access
            if user.is_superuser:
                return function(request, *args, **kw)

            if species:
                species_old_cls = species.__class__
                species.__class__ = Entry
                allow_item = None

                if item:
                    # __class__ assignment: 'Entry' object layout differs from 'ModelBase'#
                    #item_old_cls = item.__class__
                    #item = item.__class__ = Entry
                    entry_item = Entry.objects.get(pk=item.pk)
                    allow_item = user.profile.has_perm(permission, entry_item)
                    #item.__class__ = item_old_cls

                allow_species = user.profile.has_perm(permission, species)
                species._class__ = species_old_cls

                if allow_species or allow_item:
                    # Has permission
                    return function(request, *args, **kw)
            else:
                raise ValueError("Species required for permission check. Use global_permission_required to check for global permissions.")

            # No perm fallthrough
            msg = "You need permission \"{}\" to access ".format(permission)

            if item:
                msg += "item {} of ".format(item.wid)

            msg += "species {}".format(species.wid)

            return login(
                request,
                species=species,
                error=403,
                message=msg,
                force_next=request.build_absolute_uri(),
                required_perm=permission)

        return wrapper

    return decorator

def ajax_required(f):
    """
    AJAX request required decorator
    use it in your views:

    @ajax_required
    def my_view(request):
    ....

    via http://djangosnippets.org/snippets/771/
    """
    def wrap(request, *args, **kwargs):
        from django.http import HttpResponseBadRequest
        if not request.is_ajax():
            return HttpResponseBadRequest("Ajax only")
        return f(request, *args, **kwargs)
    wrap.__doc__ = f.__doc__
    wrap.__name__ = f.__name__
    return wrap

def redirect_api_request(target, target_args=list()):
    def decorator(function):
        @wraps(function)
        def wrapper(request, *args, **kw):
            format = request.GET.get('format', 'html')
            if format in ['xml', 'json']:
                return HttpResponseRedirect("%s?format=%s" % (
                    reverse(target, args=map(lambda x: kw[x], target_args)), format)
                )

            return function(request, *args, **kw)

        return wrapper

    return decorator


from typing import Optional

class SerializerConfig:
    """
    Serializer configuration options.

    Some options are not applicable for both xml or json documents.

    :param encoding: Text encoding
    :param xml_version: XML Version number (1.0|1.1)
    :param xml_declaration: Generate XML declaration
    :param pretty_print: Enable pretty output
    :param ignore_default_attributes: Ignore optional attributes with
        default values
    :param schema_location: xsi:schemaLocation attribute value
    :param no_namespace_schema_location: xsi:noNamespaceSchemaLocation
        attribute value
    """

    __slots__ = (
        "encoding",
        "xml_version",
        "xml_declaration",
        "pretty_print",
        "ignore_default_attributes",
        "schema_location",
        "no_namespace_schema_location",
    )

    def __init__(
        self,
        encoding: str = "UTF-8",
        xml_version: str = "1.0",
        xml_declaration: bool = True,
        pretty_print: bool = False,
        ignore_default_attributes: bool = False,
        schema_location: Optional[str] = None,
        no_namespace_schema_location: Optional[str] = None,
    ):
        self.encoding = encoding
        self.xml_version = xml_version
        self.xml_declaration = xml_declaration
        self.pretty_print = pretty_print
        self.ignore_default_attributes = ignore_default_attributes
        self.schema_location = schema_location
        self.no_namespace_schema_location = no_namespace_schema_location


import spice_api as spice

import os
from collections import defaultdict
import json
import libagents

import functools

path = os.path.dirname(os.path.abspath(__file__))
mangaList = defaultdict(list)
with open(path + '/config.json') as data_file:
    config = json.load(data_file)

creds = spice.init_auth(config["UserName"], config["Password"])

def catch_exception(f):
    @functools.wraps(f)
    def func(*args, **kwargs):
        count = 0
        while(count < 5):
            try:
                return f(*args, **kwargs)
            except Exception as e:
                if(count == 4):
                    print(e)
                pass

            count += 1

        print(args[0])
    return func

@catch_exception
def scrapeInfo(showId, creds):
    nameInfo = spice.search_id(showId, spice.get_medium('manga'), creds)

    mangaList[nameInfo.manga_type].append(nameInfo.title)

if __name__ == "__main__":
    agent = libagents.Agent(method=scrapeInfo, max_workers=20)
    your_list = spice.get_list(spice.get_medium('manga'), creds[0], creds)
    ids = your_list.get_ids()
    ids2 = your_list.get_status(2)
    ids = ([item for item in ids if item not in ids2])
    mangaList.clear()

    count = 1
    for id in ids:
        agent.execute_async(id, creds)

    agent.join()
    agent.finalize()
    final = ""
    for key in mangaList.keys():
        mangaList[key].sort()
        mangaList[key] = mangaList[key]
        final += "\n\n" + key + "\n" + "\n".join(mangaList[key])

    thefile = open(path + '/mangaList.txt', 'w', encoding="utf8")
    thefile.write(final[2:])


"""
Aida Muratoglu
4/6/21
M6: Poetry Slam

Writes new poems and evaluates them using natural language processing tools
and poetic lineation rules.
"""

from numpy import random
import numpy as np
from build_library import build_library
from Poem import Poem
import pyttsx3

def write_poetry():
    """Writes and evaluates 10 new poems. Returns the best one.
        Returns:
            poems (list): 10 new poems.
    """
    x = 0
    new_poems = []
    frags = call_fragments()
    verbs = get_verbs(frags)
    nouns = get_noun_phrases(frags)
    print("Writing poems...")

    while x < 10:
        num_nouns = random.randint(1, 20)
        num_verbs = random.randint(1, 20)
        # Makes a proto-poem (nouns and verbs in a list).
        list_poem = make_list_poem(verbs, nouns, num_nouns, num_verbs)
        line_breaks = random.randint(0, 15)
        y = 0
        while y < line_breaks:
            list_poem.insert(random.randint(0, len(list_poem)), "\n]")
            y += 1
        text = " ".join(list_poem)
        new_poem = Poem(text, num_nouns, num_verbs)
        new_poems.append(new_poem)
        x += 1

    return new_poems

def make_list_poem(verbs, nouns, num_nouns, num_verbs):
    """Populates list with random number of noun phrases and verbs at random
        intervals (i.e. noun noun verb noun verb verb, etc.).
        Args:
            verbs (list): verbs to use
            nouns (list): nouns to use
            num_nouns (int): number of noun phrases
            num_verbs (int): number of verbs

        Returns:
            list_poem (list): noun phrases and verbs as list
    """
    # Chooses random number of verbs and nouns for new poem
    list_poem = []
    y = 0
    while y < num_nouns + num_verbs:
            # If randint is False (0), place a verb from list
            if (random.randint(0, 1) == False):
                list_poem.append(random.choice(verbs))
            # If randint is True (1), place a noun from list
            list_poem.append(random.choice(nouns))
            y += 1
    return list_poem

def call_fragments():
    """Chooses a random number of Poems to pull from the library and returns
        them as a list.
        Returns:
            frags (list): list of Poems
    """
    print("Choosing fragments...")
    # Generates a random number
    num_frag = random.randint(1, 15)
    frags = []
    library = build_library()
    x = 0
    # Populates a list with the number of fragments (chosen at random)
    while x < num_frag:
        frags.append(random.choice(library))
        x += 1
    return frags

def get_verbs(frags):
    """Uses  natural language processing calls from the Poem class to return
        all verbs in the given fragments.
        Args:
            frags (list): list of Poems

        Returns:
            verbs (list): list of verbs from fragments
    """
    print("Getting verbs...")
    verbs = []
    x = 0
    while x < len(frags):
        verbs.extend(frags[x].get_verbs())
        x += 1
    # If no verbs, start over
    if (len(verbs) == 0):
        write_poetry()
    return verbs

def get_noun_phrases(frags):
    """Uses  natural language processing calls from the Poem class to return
        all noun phrases in the given fragments.
        Args:
            frags (list): list of Poems

        Returns:
            nouns (list): list of noun phrases from fragments
    """
    print("Getting nouns...")
    nouns = []
    x = 0
    while x < len(frags):
        nouns.extend(frags[x].get_noun_phrases())
        x += 1
    # If no nouns, start over
    if (len(nouns) == 0):
        write_poetry()
    return nouns

def get_all_scores(poems):
    """Prints all Poems with corresponding fitness scores.
        Returns:
            fitness_scores (dict): Poem objects and fitness scores.
    """
    fitness_scores = {}
    for poem in poems:
        fitness_scores[poem._evaluate()] = poem
    return fitness_scores

def choose_best_poem(poems):
    """Evaluates all poems using two parameters: ratio of noun phrases to verbs
        and line length. Returns best poem.

        Returns:
            best poem (according to fitness scores)
    """
    print("Choosing best poems...")
    fitness_scores = {}
    for poem in poems:
        fitness_scores[poem.evaluate()] = poem
        max_score = max(fitness_scores.keys())
    return fitness_scores[max_score]

def read_aloud(poem):
    """Reads poems aloud using text-to-speech software.

        Args:
            poem (str): poem to read aloud
    """
    engine = pyttsx3.init()
    engine.setProperty('rate', 150)
    engine.say(poem.text)
    engine.runAndWait()

def main():
    poems = write_poetry()
    print("Running program...")

    best = choose_best_poem(poems)

    best_or_rank = input("See best poem or rank all poems? Enter b or r: ")
    while best_or_rank != "b" and best_or_rank != "r":
        best_or_rank = input("See best poem or rank all poems? Enter b or r: ")

    print_or_save = input("Print or save the poem? Enter p or s: ")
    while print_or_save != "p" and print_or_save != "s":
        print_or_save = input("Print or save the poem? Enter p or s: ")

    if best_or_rank == "b":
        if print_or_save == "p":
            print(best)

        if print_or_save == "s":
            best.make_text_file()

    if best_or_rank == "r":
        if print_or_save == "p":
            for poem in poems:
                print(poem)
                print("Fitness score = ", poem.evaluate())

        if print_or_save == "s":
            poem_files = []
            for poem in poems:
                poem_files.append(poem.make_text_file())

    print("Reading best poem out loud...")
    # read out loud!
    read_aloud(best)

if __name__ == "__main__":
    main()