from __future__ import annotations

from dataclasses import dataclass
from typing import Optional
from leeger.model.abstract.EqualityCheck import EqualityCheck
from leeger.util.CustomLogger import CustomLogger

from leeger.util.JSONDeserializable import JSONDeserializable
from leeger.util.JSONSerializable import JSONSerializable
from leeger.util.equality import modelEquals

@dataclass(kw_only=True, eq=False)
class YearSettings(EqualityCheck, JSONSerializable, JSONDeserializable):
    __LOGGER = CustomLogger.getLogger()
    leagueMedianGames: Optional[bool] = False

    def __post_init__(self):
        if self.leagueMedianGames is None:
            self.leagueMedianGames = False

    def equals(
        self,
        otherYearSettings: YearSettings,
        *,
        ignoreIds: bool = False,
        ignoreBaseIds: bool = False,
        logDifferences: bool = False,
    ) -> bool:
        """
        Checks if *this* YearSettings is the same as the given YearSettings.
        """

        return modelEquals(
            objA=self,
            objB=otherYearSettings,
            baseFields={"leagueMedianGames"},
            parentKey="YearSettings",
            ignoreIdFields=ignoreIds,
            ignoreBaseIdField=ignoreBaseIds,
            logDifferences=logDifferences,
        )

    def __eq__(self, otherYearSettings: YearSettings) -> bool:
        self.__LOGGER.info("Use .equals() for more options when comparing YearSettings instances.")
        return self.equals(otherYearSettings=otherYearSettings)

    def toJson(self) -> dict:
        return {"leagueMedianGames": self.leagueMedianGames}

    @staticmethod
    def fromJson(d: dict) -> YearSettings:
        return YearSettings(leagueMedianGames=d.get("leagueMedianGames"))


import requests
import time

'''
****************
CONFIG
****************
'''

dataURL = "http://pulsedata57.hkustvis.org/data"
allowedPeopleLessThan = 400
attemptsMax = 100
watchKeyword = "ct1"
from barkKey import barkKey
barkURL = "https://api.day.app/" + barkKey + "/"

MSGtitle = "Eating Time"
MSGbody = "Less people than your setting"

barkURL += MSGtitle + "/" + MSGbody

'''
****************
END OF CONFIG
****************
'''

def getRawDate():
    try:
        response= requests.get(dataURL)
        if response.status_code!=200:
            return None
    except RequestException:
        return None
    raw = response.json()
    return raw['count']['zones'][watchKeyword]

def main():
    attempts = 0
    rev = getRawDate()
    while rev == None:
        attempts += 1
        time.sleep(3)
        rev = getRawDate()
        if attempts > attemptsMax:
            return -1
    if rev <= allowedPeopleLessThan:
        while rev != 200:
            URL = barkURL + ": " + str(rev) + " people."
            response = requests.get(URL)
            rev = response.json()["code"]

if __name__ == "__main__":
    main()


import json

from fastapi import FastAPI

from fastapi.middleware.cors import CORSMiddleware

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Allows all origins
    allow_credentials=True,
    allow_methods=["*"],  # Allows all methods
    allow_headers=["*"],  # Allows all headers
)

@app.get("/")
async def root():
    # # get food data
    with open("foods.json") as f:
        data = json.load(f)
    return data


from flask import Flask, request, jsonify
from flask_cors import CORS
from recommendations import get_recommendations

app = Flask(__name__)
# CORS(app)
CORS(app, resources={r"/api/*": {"origins": "*"}})

@app.route('/', methods=['GET'])
def home():
    return "Welcome to the recommendation service!"

@app.route('/recommendations', methods=['GET'])
def recommendations():
    user_id = request.args.get('user_id', type=int)
    recommended_professionals = get_recommendations(user_id)
    return jsonify(recommended_professionals.tolist())

@app.after_request
def after_request(response):
    response.headers.add('Access-Control-Allow-Origin', '*')
    response.headers.add('Access-Control-Allow-Headers',
                         'Content-Type,Authorization')
    response.headers.add('Access-Control-Allow-Methods',
                         'GET,PUT,POST,DELETE,OPTIONS')
    return response

if __name__ == '__main__':
    app.run(debug=True)


import os
import sys
import argparse
import glob
import pprint

import pandas as pd

def main(args):
    metadataDF = pd.read_csv(args.gtdb_metadata, header=0, low_memory=False)
    metadataDF["accession"] = metadataDF["accession"].str.replace("GB_", "").str.replace("RS_", "")
    # read in info about version errors encountered during downloads
    version_err = pd.read_csv(args.version_errors_log, header=None, names = ["accession", "prev_version", "full_name"])
    verrD = pd.Series(version_err["prev_version"].values,index=version_err["accession"]).to_dict()
   # replace accessions to new accs
    metadataDF["accession"].replace(verrD, inplace=True)
    import pdb;pdb.set_trace()
    metadataDF.to_csv(args.output_csv, index=False)

def cmdline(sys_args):
    "Command line entry point w/argparse action."
    p = argparse.ArgumentParser()
    p.add_argument("--gtdb-metadata",  default = "/group/ctbrowngrp/gtdb/gtdb-rs202.metadata.csv.gz")
    p.add_argument("--version-errors-log", default="/home/ntpierce/sourmash_databases/output.gtdb-databases/accession_version_errors.csv")
    p.add_argument("--output-csv", default="/group/ctbrowngrp/gtdb/gtdb-rs202.metadata.v2.csv.gz")
    args = p.parse_args()
    return main(args)

if __name__ == '__main__':
    returncode = cmdline(sys.argv[1:])
    sys.exit(returncode)


import collections
import numpy as np
from typing import List

#cant binary search it. it requires two-pointer.
#There is no correlation between the middle value and the left and right pointers.
class Solution:
    def twoSum(self, numbers: List[int], target: int) -> List[int]:
        l, r = 0, len(numbers)-1
        while l < r:
            mid = l + (r-l)//2
            if numbers[mid] + numbers[mid+1] == target:
                return [mid+1, mid+2]
            if numbers[mid-1] + numbers[mid] == target:
                return [mid, mid+1]
            if numbers[l] + numbers[r]==target and l!=r:
                return [l+1,r+1]
            if numbers[r] > target:
                r-=1
            elif numbers[mid]+numbers[r] < target:
                l = mid + 1
            else:
                r = mid
            print(l,r,mid)
        print('s')
        return sorted([l+1,r+1])

s = Solution()
#2,3
print(s.twoSum([5, 25, 75], 100))
#1,2
print(s.twoSum([2,7,11,15], 9))
#2,3
print(s.twoSum([2,3,4], 6))
#1,2
print(s.twoSum([0,0,3,4], 0))
#3,6
print(s.twoSum([3,24,50,79,88,150,345],200))


#!/usr/bin/env python

"""
@Author: gorywong
@Date: 2019-01-23 15:15:01
@Software: Visual Studio Code
@Last Modified by: gorywong
@Last Modified time: 2019-01-23 15:15:01
@Description:
"""
import logging
from abc import abstractmethod

from django.db import models
from django.urls import reverse
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.functional import cached_property
from django.utils.timezone import now
from ckeditor_uploader.fields import RichTextUploadingField

from lbyledu.utils import cache_decorator, cache
from users.models import UserGroup, UserProfile
from organization.models import Organization

logger = logging.getLogger(__name__)

# Create your models here.
class BaseModel(models.Model):
    id = models.AutoField(primary_key=True)
    created_time = models.DateTimeField(default=now, verbose_name="创建时间")
    last_mod_time = models.DateTimeField(default=now, verbose_name="修改时间")

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)

    class Meta:
        abstract = True

    @abstractmethod
    def get_absolute_url(self):
        pass

class Document(BaseModel):
    title = models.CharField(max_length=200, verbose_name="标题")
    author = models.ForeignKey(UserProfile, on_delete=models.CASCADE, verbose_name="发布人")
    content = RichTextUploadingField(verbose_name="正文")
    receiver = models.ManyToManyField(UserProfile, blank=True, related_name="documents", verbose_name="收件人")
    receiver_group = models.ForeignKey(UserGroup, on_delete=models.CASCADE, verbose_name="收件用户组")
    checked_receiver = models.ManyToManyField(UserProfile, blank=True, related_name="checked_documents", verbose_name="已签收的用户")
    number = models.IntegerField(unique=True, verbose_name="编号")

    class Meta:
        verbose_name = "公文"
        verbose_name_plural = verbose_name
        ordering = ['-created_time']
        get_latest_by = 'id'

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse("office:document_detail", kwargs={"number": self.number})

class OrganizationAdmins(BaseModel):
    organization = models.ForeignKey(Organization, on_delete=models.CASCADE, verbose_name="单位")
    admin  = models.ForeignKey(UserProfile, on_delete=models.CASCADE, verbose_name="管理员")

    class Meta:
        verbose_name = "网站管理员"
        verbose_name_plural = verbose_name


#!/usr/bin/env python3

"""A simple JAX process launcher for multiple devices on a single host.

You must import jax_local_cluster somewhere inside the script you are launching.
"""

import argparse
from functools import partial
import os
import signal
import socketserver
from subprocess import Popen, TimeoutExpired
import sys

import jax
import jax._src as _src

error = partial(print, file=sys.stderr)

class LocalCluster(_src.clusters.ClusterEnv):
    @classmethod
    def is_env_present(cls):
        return "JAX_COORDINATOR_ADDRESS" in os.environ

    @classmethod
    def get_coordinator_address(cls):
        return os.environ["JAX_COORDINATOR_ADDRESS"]

    @classmethod
    def get_process_count(cls):
        return int(os.environ["JAX_PROCESS_COUNT"])

    @classmethod
    def get_process_id(cls):
        return int(os.environ["JAX_PROCESS_ID"])

    @classmethod
    def get_local_process_id(cls):
        return int(os.environ["JAX_LOCAL_PROCESS_ID"])

def get_free_port():
    with socketserver.TCPServer(("127.0.0.1", 0), None) as s:
        return s.server_address[1]

def signal_and_wait(signum, procs, ctx, timeout=None):
    for proc in procs:
        proc.send_signal(signum)
    for i, proc in enumerate(procs):
        ctx["i"] = i
        proc.wait(timeout)
    ctx["i"] = None

def interactive_shutdown(procs):
    ctx = {"i": None}
    try:
        signal_and_wait(signal.SIGINT, procs, ctx)
    except KeyboardInterrupt:
        try:
            error(
                f"Process {ctx['i']} (pid {procs[ctx['i']].pid}) did not exit on SIGINT, trying SIGTERM"
            )
            signal_and_wait(signal.SIGTERM, procs, ctx, timeout=1)
        except (KeyboardInterrupt, TimeoutExpired):
            error(
                f"Process {ctx['i']} (pid {procs[ctx['i']].pid}) did not exit on SIGTERM, trying SIGKILL"
            )
            for proc in procs:
                proc.kill()

class TerminationHandler:
    def __init__(self, procs, verbose):
        self.procs = procs
        self.verbose = verbose
        self.was_called = False

    def __call__(self, signum, frame):
        self.was_called = True
        if self.verbose:
            error("SIGTERM received, shutting down")
        try:
            signal_and_wait(signal.SIGTERM, self.procs, {}, timeout=1)
        except TimeoutExpired:
            if self.verbose:
                error("SIGTERM timed out, sending SIGKILL")
            for proc in self.procs:
                proc.kill()
        raise KeyboardInterrupt

def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument(
        "-n",
        type=int,
        default=0,
        help="Number of processes to launch (default: one per local device)",
    )
    p.add_argument(
        "--port",
        type=int,
        default=0,
        help="Port to use for the coordinator (default: a free port)",
    )
    p.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
    p.add_argument("command", type=str, nargs=argparse.REMAINDER, help="Command to run")
    args = p.parse_args()

    if not args.command:
        p.print_help()
        sys.exit(1)

    n = args.n if args.n else jax.local_device_count()
    if args.verbose:
        error(f"Launching {n} processes")
    port = args.port if args.port else get_free_port()
    if args.verbose:
        error(f"Using port {port} for coordinator")

    procs = []
    sigterm_handler = TerminationHandler(procs, args.verbose)
    signal.signal(signal.SIGTERM, sigterm_handler)

    try:
        for i in range(n):
            env = os.environ.copy()
            env.pop("OMPI_MCA_orte_hnp_uri", None)
            env.pop("SLURM_JOB_ID", None)
            env["JAX_COORDINATOR_ADDRESS"] = f"127.0.0.1:{port}"
            env["JAX_PROCESS_COUNT"] = str(n)
            env["JAX_PROCESS_ID"] = str(i)
            env["JAX_LOCAL_PROCESS_ID"] = str(i)
            proc = Popen(args.command, env=env)
            if args.verbose:
                error(f"Launched process {i} (pid {proc.pid})")
            procs.append(proc)
        for proc in procs:
            proc.wait()
    except KeyboardInterrupt:
        pass
    finally:
        if not sigterm_handler.was_called:
            interactive_shutdown(procs)
        if args.verbose:
            error("All processes terminated")

if __name__ == "__main__":
    main()


# coding:utf-8
from net import VOCDataset, TrainPipeline
from utils.augmentation_utils import SSDAugmentation

if __name__ == '__main__':
    # load dataset
    root = 'data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007'
    dataset = VOCDataset(root, 'trainval', SSDAugmentation(), True)

    # train config
    config = {
        'dataset': dataset,
        'n_classes': len(dataset.classes)+1,
        'vgg_path': 'model/vgg16_reducedfc.pth',
        'lr': 5e-4,
        'batch_size': 8,
        'num_workers': 4,
        'start_iter': 0,
        'max_iters': 120000,
        'warm_up_iters': 500,
        'lr_steps': (80000, 100000, 120000),
        'save_frequency': 5000,
    }

    # train
    train_pipeline = TrainPipeline(**config)
    train_pipeline.train()


import os
import copy

from google.appengine.api import users
from google.appengine.ext import ndb
from google.appengine.ext.ndb import msgprop
from protorpc import messages

from google.appengine.api import mail

from jinja2 import Environment, FileSystemLoader
import webapp2

debug = os.environ.get('SERVER_SOFTWARE', '').startswith('Dev')

JINJA_ENVIRONMENT = Environment(
#    loader=PackageLoader('estimate', 'templates'),
    loader=FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

def format_currency(value):
    return "${:,.2f}".format(value)

JINJA_ENVIRONMENT.filters.update({
    'format_currency': format_currency,
})

def extra_context(handler, context=None):
    """
    Adds extra context.
    """

    context = context or {}

    return dict({'request': handler.request}, **context)

class Shape(messages.Enum):
    L = 10
    U = 20
    R = 30
    O = 0

class CountertopMaterial(ndb.Model):
    type = ndb.StringProperty(indexed=False)
    cost_per_slab = ndb.FloatProperty(indexed=False)
    color = ndb.StringProperty()
    slab_size_sqft = ndb.FloatProperty(indexed=False)
    date_created = ndb.DateTimeProperty(auto_now_add=True)
    labor_cost_per_sqft = ndb.FloatProperty(indexed=False)
    image_filename = ndb.StringProperty(indexed=False)
"""
# Group 1
cm1 = CountertopMaterial()
cm1.populate(type="granite", cost_per_slab=800, color="Cecilia Light", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group1_cecilia_light.JPG")
cm1.put()

cm2 = CountertopMaterial()
cm2.populate(type="granite", cost_per_slab=800, color="Fiorito", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group1_fiorito.JPG")
cm2.put()

cm3 = CountertopMaterial()
cm3.populate(type="granite", cost_per_slab=800, color="Ouro Brazil", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group1_ouro_brazil.JPG")
cm3.put()

cm4 = CountertopMaterial()
cm4.populate(type="granite", cost_per_slab=800, color="Tan Brown", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group1_tan_brown.JPG")
cm4.put()

# Group 2
cm5 = CountertopMaterial()
cm5.populate(type="granite", cost_per_slab=1100, color="Giblee", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group2_giblee.JPG")
cm5.put()

cm6 = CountertopMaterial()
cm6.populate(type="granite", cost_per_slab=1100, color="Juparana Wave", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group2_juparana_wave.JPG")
cm6.put()

cm7 = CountertopMaterial()
cm7.populate(type="granite", cost_per_slab=1100, color="Kashmire Cream", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group2_kashmire_cream.JPG")
cm7.put()

cm8 = CountertopMaterial()
cm8.populate(type="granite", cost_per_slab=1100, color="Sapphire", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group2_sapphire.JPG")
cm8.put()

# Group 3
cm9 = CountertopMaterial()
cm9.populate(type="granite", cost_per_slab=1900, color="Copenhagen", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group3_copenhagen.JPG")
cm9.put()

cm10 = CountertopMaterial()
cm10.populate(type="granite", cost_per_slab=1900, color="Golden Crystal", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group3_golden_crystal.JPG")
cm10.put()

cm11 = CountertopMaterial()
cm11.populate(type="granite", cost_per_slab=1900, color="Thyphoon Bordeaux", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group3_thyphoon_bordeaux.JPG")
cm11.put()

cm12 = CountertopMaterial()
cm12.populate(type="granite", cost_per_slab=1900, color="White Ice", slab_size_sqft=45, labor_cost_per_sqft=50, image_filename="group3_white_ice.JPG")
cm12.put()
"""

class Client(ndb.Model):
    email = ndb.StringProperty(indexed=False)
    phone = ndb.StringProperty(indexed=False)
    street1 = ndb.StringProperty(indexed=False)
    street2 = ndb.StringProperty(indexed=False)
    zipcode = ndb.StringProperty(indexed=False)
    city = ndb.StringProperty(indexed=False)
    name = ndb.StringProperty(indexed=False)
    message = ndb.TextProperty(indexed=False)
    date_created = ndb.DateTimeProperty(auto_now_add=True)

class CountertopShape(ndb.Model):
    A = ndb.FloatProperty(indexed=False)
    B = ndb.FloatProperty(indexed=False)
    C = ndb.FloatProperty(indexed=False)
    D = ndb.FloatProperty(indexed=False)
    E = ndb.FloatProperty(indexed=False)
    F = ndb.FloatProperty(indexed=False)
    shape = msgprop.EnumProperty(Shape, required=True)
    sqft = ndb.ComputedProperty(lambda self: self.compute_sqft())

    def compute_sqft(self):
        if self.shape == Shape.L:
            return (self.A * self.B + self.C * self.D) / 144.0
        if self.shape == Shape.U:
            return (self.A * self.B + self.F * (self.C - self.E) + self.C * self.D) / 144.0
        if self.shape == Shape.R:
            return (self.A * self.B) / 144.0
        return 0

    def get_sides(self):
        if self.shape == Shape.L:
            return (self.A, self.B, self.C, self.D)
        elif self.shape == Shape.R:
            return (self.A, self.B)
        elif self.shape == Shape.U:
            return (self.A, self.B, self.C, self.D, self.E, self.F)
        return ()

    def get_name(self):
        if self.shape == Shape.L:
            return 'L-shape'
        elif self.shape == Shape.U:
            return 'U-shape'
        elif self.shape == Shape.R:
            return 'R-shape'
        return 'Shape: other'

    def get_path(self):
        if self.shape == Shape.L:
            return '/lshape'
        if self.shape == Shape.U:
            return '/ushape'
        if self.shape == Shape.R:
            return '/rshape'
        return '/contact'

class Estimate(ndb.Model):
    """Sub model for representing an author."""
    client = ndb.StructuredProperty(Client)
    countertop_shapes = ndb.StructuredProperty(CountertopShape, repeated = True)
    total_sqft = ndb.FloatProperty(indexed=False)
    total_price = ndb.FloatProperty(indexed=False)
    date_created = ndb.DateTimeProperty(auto_now_add=True)
    need_attention = ndb.BooleanProperty()
    ip = ndb.StringProperty(indexed=False)
    labor_cost = ndb.FloatProperty(indexed=False)
    material = ndb.StructuredProperty(CountertopMaterial) # one for all pieces
    material_cost = ndb.FloatProperty(indexed=False)

    def compute_total_sqft(self):
        total = 0
        for shape in self.countertop_shapes:
            total += shape.sqft
        return total

    def compute_material_cost(self):
        return (int(self.compute_total_sqft() / self.material.slab_size_sqft) + 1) * self.material.cost_per_slab

    def compute_total_price(self):
        return self.compute_material_cost() + self.compute_labor_cost()

    def compute_labor_cost(self):
        return self.compute_total_sqft() * self.material.labor_cost_per_sqft

    def compute_number_of_slabs(self):
        return int(self.compute_total_sqft() / self.material.slab_size_sqft) + 1

class Order(ndb.Model):
    client = ndb.StructuredProperty(Client)
    estimate = ndb.StructuredProperty(Estimate)
    date_created = ndb.DateTimeProperty(auto_now_add=True)

class MainPage(webapp2.RequestHandler):

    def get(self):
        # estimates = Estimate.query().order(-Estimate.date_created)
        """
        user = users.get_current_user()
        if user:
            url = users.create_logout_url(self.request.uri)
            url_linktext = 'Logout'
        else:
            url = users.create_login_url(self.request.uri)
            url_linktext = 'Login'
        """
        template = JINJA_ENVIRONMENT.get_template('index.html')
        self.response.write(template.render(**extra_context(self)))

class TempPage(webapp2.RequestHandler):

    def get(self):
        # estimates = Estimate.query().order(-Estimate.date_created)
        """
        user = users.get_current_user()
        if user:
            url = users.create_logout_url(self.request.uri)
            url_linktext = 'Logout'
        else:
            url = users.create_login_url(self.request.uri)
            url_linktext = 'Login'
        """
        template = JINJA_ENVIRONMENT.get_template('temp.html')
        self.response.write(template.render(**extra_context(self)))

class ShapeRevertHandler(webapp2.RequestHandler):

    def get(self):
        eid = self.request.get('eid')
        if not eid:
            self.redirect('/')

        key = ndb.Key(urlsafe=eid)
        estimate = key.get()

        if len(estimate.countertop_shapes) > 0:
            path = estimate.countertop_shapes[-1].get_path()
            new_shapes = copy.copy(estimate.countertop_shapes[:-1])
            estimate.countertop_shapes = new_shapes
            estimate.put()

            self.redirect('%s?eid=%s' % (path, estimate.key.urlsafe()))
        else:
            self.redirect('/')

class ShapeHandler(webapp2.RequestHandler):
    SHAPED = {'/lshape': 'lshape.html', '/rshape': 'rshape.html', '/ushape': 'ushape.html'}

    def get(self):
        name = self.request.path
        template = JINJA_ENVIRONMENT.get_template(self.SHAPED.get(name))
        self.response.write(template.render(**extra_context(self)))

    def get_shape(self):
        if self.request.path == '/lshape':
            return Shape.L
        if self.request.path == '/rshape':
            return Shape.R
        if self.request.path == '/ushape':
            return Shape.U
        return Shape.O

    def post(self):
        side_A = self.request.get('side_A')
        side_B = self.request.get('side_B')
        side_C = self.request.get('side_C')
        side_D = self.request.get('side_D')
        side_E = self.request.get('side_E')
        side_F = self.request.get('side_F')
        eid = self.request.get('eid')
        shape = self.get_shape()

        def _do_redirect():
            self.redirect( "{0}?eid={1}&side_A={2}&side_B={3}&side_C={4}&side_D={5}".
                           format(self.request.path, eid, side_A, side_B, side_C, side_D, side_E, side_F))

        if not unicode.isdigit(side_A) or not unicode.isdigit(side_B):
            _do_redirect()
            return

        if shape == Shape.L or shape == Shape.U:
            if not unicode.isdigit(side_C) or not unicode.isdigit(side_D):
                _do_redirect()

        if shape == Shape.U:
            if not unicode.isdigit(side_E) or not unicode.isdigit(side_F):
                _do_redirect()

        # everything is fine
        if eid:
            key = ndb.Key(urlsafe=eid)
            estimate = key.get()
        else:
            estimate = Estimate()

        # estimate.material = CountertopMaterial.query()[0]
        piece = CountertopShape(shape=shape)
        # material = CountertopMaterial().query(CountertopMaterial.color == 'black').fetch(1)[0]
        # estimate.material = material
        if side_A: piece.A = float(side_A)
        if side_B: piece.B = float(side_B)
        if side_C: piece.C = float(side_C)
        if side_D: piece.D = float(side_D)
        if side_E: piece.E = float(side_E)
        if side_F: piece.F = float(side_F)

        estimate.countertop_shapes.append(piece)
        estimate.total_sqft = estimate.compute_total_sqft()
        # estimate.labor_cost = estimate.compute_labor_cost()
        # estimate.total_price = estimate.compute_total_price()
        estimate.ip = self.request.remote_addr
        estimate.put()
        self.redirect('/material?eid=%s' % estimate.key.urlsafe())

class TotalHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('total.html')
        eid = ndb.Key(urlsafe=self.request.get('eid'))
        mid = ndb.Key(urlsafe=self.request.get('mid'))
        estimate = eid.get()
        material = mid.get()
        estimate.material = material
        estimate.put()
        template_values = { 'estimate': estimate }

        self.response.write(template.render(**extra_context(self, template_values)))

    def post(self):
        side_A = self.request.get('side_A')
        side_B = self.request.get('side_B')
        side_C = self.request.get('side_C')
        side_D = self.request.get('side_D')
        eid = self.request.get('eid')
        key = ndb.Key(urlsafe=eid)

        if not unicode.isdigit(side_A) or not unicode.isdigit(side_B) or\
        not unicode.isdigit(side_C) or not unicode.isdigit(side_D):
            self.redirect('/lshape?eid=%s&side_A=%s&side_B=%s&side_C=%s&side_D=%s' % (eid, side_A, side_B, side_C, side_D))
            return

        # everything is fine
        if key:
            estimate = key.get()
        else:
            estimate = Estimate()

        # estimate.material = CountertopMaterial.query()[0]
        shape = CountertopShape(shape=Shape.L)
        # material = CountertopMaterial().query(CountertopMaterial.color == 'black').fetch(1)[0]
        # estimate.material = material
        shape.A = float(side_A)
        shape.B = float(side_B)
        shape.C = float(side_C)
        shape.D = float(side_D)

        estimate.countertop_shapes.append(shape)
        estimate.total_sqft = estimate.compute_total_sqft()
        # estimate.labor_cost = estimate.compute_labor_cost()
        # estimate.total_price = estimate.compute_total_price()
        estimate.ip = self.request.remote_addr
        estimate.put()
        self.redirect('/material?eid=%s' % estimate.key.urlsafe())

class MaterialHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('material.html')
        l = CountertopMaterial.query()
        template_values = { 'materials': l }

        self.response.write(template.render(**extra_context(self, template_values)))
        return

class EstimateHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('estimate.html')
        self.response.write(template.render(**extra_context(self)))

class AboutHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('about.html')
        self.response.write(template.render(**extra_context(self)))

class AdminHandler(webapp2.RequestHandler):
    def get(self):
        estimates = Estimate.query(Estimate.need_attention == True)

        user = users.get_current_user()
        if not user:
            url = users.create_login_url(self.request.uri)
            self.redirect(url)
            return

        """
        if user.nickname() != "admin@admin.com":
            self.response.write("Not authorized")
            return
        """

        url = users.create_logout_url(self.request.uri)
        url_linktext = 'Logout'

        template_values = {
            'user': user,
            'estimates': estimates,
            'url': url,
            'url_linktext': url_linktext,
            'user': user
        }
        template = JINJA_ENVIRONMENT.get_template('admin.html')
        self.response.write(template.render(template_values))

class ContactHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('contact_other.html')
        self.response.write(template.render(**extra_context(self)))

    def post(self):
        name = self.request.get('name')
        phone = self.request.get('phone')
        email = self.request.get('email')
        message = self.request.get('message')
        eid = self.request.get('eid')

        client = Client()
        client.name = name
        client.email = email
        client.phone = phone
        client.message = message
        client.put()

        if eid:
            key = ndb.Key(urlsafe=eid)

            estimate = key.get()
            estimate.client = client
            estimate.need_attention = True
            estimate.total_price = estimate.compute_total_price()
            estimate.total_sqft = estimate.compute_total_sqft()
            estimate.material_cost = estimate.compute_material_cost()
            estimate.labor_cost = estimate.compute_labor_cost()

            estimate.put()

            template = JINJA_ENVIRONMENT.get_template('email.html')
            template_values = {
                               'client': client,
                               'estimate': estimate }
            html = template.render(**extra_context(self, template_values))
        else:
            template = JINJA_ENVIRONMENT.get_template('client_details.html')
            template_values = { 'client': client }

            html = template.render(**extra_context(self, template_values))

        message = mail.EmailMessage(sender="grybkin@gmail.com",
                            subject="New Client")

        message.to = "sergey@domus-surfaces.com"
        message.cc = "sergek@russianamericanmedia.com"
        message.html = html
        message.send()

        self.redirect('/thankyou')

class ThankyouHandler(webapp2.RequestHandler):
    def get(self):
        template = JINJA_ENVIRONMENT.get_template('thankyou.html')
        self.response.write(template.render(**extra_context(self)))

application = webapp2.WSGIApplication([
#    ('/', TempPage),
    ('/', MainPage),
    ('/estimate', EstimateHandler),
    ('/about', AboutHandler),
    ('/lshape', ShapeHandler),
    ('/ushape', ShapeHandler),
    ('/rshape', ShapeHandler),
    ('/shape', ShapeRevertHandler),
    ('/contact', ContactHandler),
    ('/admin', AdminHandler),
    ('/material', MaterialHandler),
    ('/total', TotalHandler),
    ('/thankyou', ThankyouHandler),
#    ('/rest/.*', rest.Dispatcher),
], debug=debug)

# configure the rest dispatcher to know what prefix to expect on request urls
# rest.Dispatcher.base_url = "/rest"

# add all models from the current module, and/or...
# rest.Dispatcher.add_models_from_module(__name__)
# add all models from some other module, and/or...
# rest.Dispatcher.add_models_from_module(my_model_module)
# add specific models
"""
rest.Dispatcher.add_models({
#  "estimate": Estimate,
  "material": CountertopMaterial})
# add specific models (with given names) and restrict the supported methods

rest.Dispatcher.add_models({
  'material' : (CountertopMaterial, rest.READ_ONLY_MODEL_METHODS),
#  'estimate' : (Estimate, ['GET', 'POST', 'PUT']),
#  'cache' : (CacheModel, ['GET_METADATA', 'GET', 'DELETE']
})

# optionally use custom authentication/authorization
# rest.Dispatcher.authenticator = MyAuthenticator()
# rest.Dispatcher.authorizer = MyAuthorizer()

"""


import torch
import torch.nn as nn
from torch.utils import data

from datasets import Dataset
from networks import TransformerNet

def train(training_input, training_labels, num_unique_tokens, sequence_length):
    # Loader Parameters
    params = {'batch_size': 32,
              'shuffle': True,
              'num_workers': 12}
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    # Create Batch Generator
    training_dataset = Dataset(training_input, training_labels)
    training_dataloader = torch.utils.data.DataLoader(training_dataset, **params)

    # Define network, loss function and optimizer
    net = TransformerNet(embedding_dim=500, num_mappings=num_unique_tokens,
                         num_heads=4, num_blocks=3, seq_length=sequence_length)
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(net.parameters(), lr=1e-4, weight_decay=1e-3)

    # Define number of epochs
    num_epochs = 10
    print_batch_step = 100
    best_epoch_loss = 1000

    # Start training
    net = net.to(device)
    net.train(mode=True)
    for epoch in range(num_epochs):

        running_loss = 0
        epoch_loss = 0

        for i, data_ in enumerate(training_dataloader, 1):

            inputs, labels = data_[0].to(device), data_[1].to(device)

            optimizer.zero_grad()
            outputs = net(inputs)

            # flatten sequence dim to batch dim before Softmax + CE
            batch, seq, num_classes = outputs.size()
            outputs = outputs.view(batch * seq, num_classes)
            labels = labels.view(batch * seq)
            loss = criterion(outputs, labels)

            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            epoch_loss += loss.item() * params['batch_size']
            if i % print_batch_step == 0:
                print('[%d, %5d] loss: %.3f' % (epoch + 1, i, running_loss / print_batch_step))
                running_loss = 0

        epoch_num = epoch + 1
        average_epoch_loss = epoch_loss / len(training_dataset)
        print(f"Epoch {epoch_num}: average loss is {average_epoch_loss}")

        if average_epoch_loss < best_epoch_loss:
            net.save_checkpoint(epoch_num, average_epoch_loss, optimizer.state_dict())
            best_epoch_loss = average_epoch_loss

    print('Finished Training')


import serial
import time

ser = serial.Serial('/dev/ttyAMA0', 115200)

class serial_communication:
    def __init__(self):
        ready = self.handshake()
        while ready == False:
            print('no handshake')
            ready = self.handshake()
        while ready:
            global rcv
            global los
            data = self.readlineCR()
            if (data == '0'):
                print(rcv)
                los = len(rcv)
            if (data == '1'):
                print(rcv)
                if(los == int(rcv)):
                    print('size is correct')
                else :
                    print ('size is wrong')
                los = ""
            if (data == '2'):
                print('checksum is {}'.format(rcv))

    def handshake(self):
        ser.write('0'.encode())
        ack1 = ser.read().decode('ascii')
        if ack1 == '1':
            ser.write('1'.encode())
            print('handshake pass')
            return True
        else:
            print("\r\n\Handshake not completed!")
            return False

    def readlineCR(self):
        global rcv
        rcv = ""
        checksum = 0
        while True:
            ch = ser.read().decode('ascii')
            if(ch == 'd'):
                return '0'
            if(ch == 's'):
                return '1'
            if (ch == 'c'):
                return '2'
            rcv += ch

my_serial_communication = serial_communication()


import csv
import matplotlib.pyplot as plt
import sqlite3
import requests
import json
import os

#create table that will contain data for number of positive COVID-19 cases in DC given a date
def create_covid_cases_table(cur, conn):
    cur.execute("CREATE TABLE IF NOT EXISTS Cases (Date INTEGER, Cases INTEGER)")
    conn.commit()

#get the request url to access the COVID tracking API based on the date
def get_request_url(date):
    base_url = "https://api.covidtracking.com"
    request_url = base_url + "/v1/states/dc/{}.json" .format(date)
    return request_url

#gets data from the API only if data does not already exist in Cases table
def get_covid_cases_data(cur, conn, date):
    #gets request url
    url = get_request_url(date)

    #checking to see if date is already in table
    cur.execute("SELECT * FROM Cases WHERE Date = ?", (date, ))
    exists = cur.fetchone()

    #if data not already in table, try grabbing data and returning # of positive cases
    if exists == None:
        try:
            response = requests.get(url)
            json_data = json.loads(response.text)
            cases = json_data.get('positive')
            return cases
    #if exception occurs, prints exception
        except:
            print("Exception")
            return None
    #lets me know if data does already exist in table
    else:
        print("Data already exists")
        return None

#adds data to database table Cases 25 items at a time for the first 100 items
def add_data_to_database(cur, conn, dates, cases):
    cur.execute("SELECT * FROM Cases")
    entries = cur.fetchall()
    if len(entries) == 0:
        for i in range(25):
            cur.execute("INSERT INTO Cases (Date, Cases) Values (?,?)", (dates[i], cases[i],))
            conn.commit()
    elif len(entries) == 25:
        for i in range(25,50):
            cur.execute("INSERT INTO Cases (Date, Cases) Values (?,?)", (dates[i], cases[i],))
            conn.commit()
    elif len(entries) == 50:
        for i in range(50,75):
            cur.execute("INSERT INTO Cases (Date, Cases) Values (?,?)", (dates[i], cases[i],))
            conn.commit()
    elif len(entries) == 75:
        for i in range(75,100):
            cur.execute("INSERT INTO Cases (Date, Cases) Values (?,?)", (dates[i], cases[i],))
            conn.commit()
    elif len(entries) == 100:
        for i in range(100, len(dates)):
            cur.execute("INSERT INTO Cases (Date, Cases) Values (?,?)", (dates[i], cases[i],))
            conn.commit()

def main():

#the following lists are just creating lists of dates in the appropriate format for the API
    march_dates_list = []
    for i in range (7,32):
        if i < 10:
            date_string = "2020030" + str(i)
        else:
            date_string = "202003" + str(i)
        march_dates_list.append(date_string)

    april_dates_list1 = []
    for i in range(1,26):
        if i < 10:
            date_string = "2020040" + str(i)
        else:
            date_string = "202004" + str(i)
        april_dates_list1.append(date_string)

    april_dates_list2 = []
    for i in range(26,31):
        date_string = "202004" + str(i)
        april_dates_list2.append(date_string)

    may_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020050" + str(i)
        else:
            date_string = "202005" + str(i)
        may_dates_list1.append(date_string)

    may_dates_list2 = []
    for i in range(26,32):
        date_string = "202005" + str(i)
        may_dates_list2.append(date_string)

    june_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020060" + str(i)
        else:
            date_string = "202006" + str(i)
        june_dates_list1.append(date_string)

    june_dates_list2 = []
    for i in range(26,31):
        date_string = "202006" + str(i)
        june_dates_list2.append(date_string)

    july_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020070" + str(i)
        else:
            date_string = "202007" + str(i)
        july_dates_list1.append(date_string)

    july_dates_list2 = []
    for i in range(26,32):
        date_string = "202007" + str(i)
        july_dates_list2.append(date_string)

    august_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020080" + str(i)
        else:
            date_string = "202008" + str(i)
        august_dates_list1.append(date_string)

    august_dates_list2 = []
    for i in range(26,32):
        date_string = "202008" + str(i)
        august_dates_list2.append(date_string)

    september_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020090" + str(i)
        else:
            date_string = "202009" + str(i)
        september_dates_list1.append(date_string)

    september_dates_list2 = []
    for i in range(26,31):
        date_string = "202009" + str(i)
        september_dates_list2.append(date_string)

    october_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020100" + str(i)
        else:
            date_string = "202010" + str(i)
        october_dates_list1.append(date_string)

    october_dates_list2 = []
    for i in range(26,32):
        date_string = "202010" + str(i)
        october_dates_list2.append(date_string)

    november_dates_list1 = []
    for i in range (1,26):
        if i < 10:
            date_string = "2020110" + str(i)
        else:
            date_string = "202011" + str(i)
        november_dates_list1.append(date_string)

    november_dates_list2 = []
    for i in range(26,31):
        date_string = "202011" + str(i)
        november_dates_list2.append(date_string)

    december_dates_list = []
    for i in range(1,8):
        date_string = "2020120" + str(i)
        december_dates_list.append(date_string)

#this adds all of the lists together into one final list with every date from March 7th-December 7th in the appropriate format for the COVID API
    total_dates = []
    total_dates = [march_dates_list,april_dates_list1,april_dates_list2,may_dates_list1,may_dates_list2,june_dates_list1,june_dates_list2,july_dates_list1,july_dates_list2,august_dates_list1,august_dates_list2,september_dates_list1,september_dates_list2,october_dates_list1,october_dates_list2,november_dates_list1,november_dates_list2,december_dates_list]
    flat_dates_list = []
    for sublist in total_dates:
        for item in sublist:
            flat_dates_list.append(item)

#getting the current working directory, connecting to the database, and creating the Cases table
    path = os.path.dirname(os.path.abspath(__file__))
    conn = sqlite3.connect(path+ '/' + "covid_data.db")
    cur = conn.cursor()
    create_covid_cases_table(cur,conn)

#getting a list of the positive cases in chronological order to be added to the table
    cases_list = []
    for date in flat_dates_list:
        data = get_covid_cases_data(cur,conn,date)
        cases_list.append(data)

#adding data to Cases table based on dates list and postiive cases list
    add_data_to_database(cur,conn,flat_dates_list,cases_list)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
"""Monitor for new devices/clients on the network.

The network dhcp server is queried for currently known DHCP clients.
Any new clients are identified and a notification is sent.
"""

#==========================================================
#
#  Chris Nelson, 2020 - 2023
#
# 3.0 230215 - Converted to package format, updated to cjnfuncs 2.0
# ...
# 0.1 200426 - New
#
#==========================================================

import argparse
import subprocess
import sys
import re
import time
import datetime
import os.path
import collections
# import html       # Needed for alternate MACOUI lookup implementation
import sqlite3
import requests
import signal
from lxml import html as _html

try:
    import importlib.metadata
    __version__ = importlib.metadata.version(__package__ or __name__)
except:
    try:
        import importlib_metadata
        __version__ = importlib_metadata.version(__package__ or __name__)
    except:
        __version__ = "3.0 X"

# from cjnfuncs.cjnfuncs import set_toolname, setup_logging, logging, config_item, getcfg, mungePath, deploy_files, timevalue, retime, requestlock, releaselock,  snd_notif, snd_email
from cjnfuncs.cjnfuncs import set_toolname, logging, config_item, getcfg, mungePath, deploy_files, timevalue, snd_notif

# Configs / Constants
TOOLNAME            = "routermonitor"
CONFIG_FILE         = "routermonitor.cfg"
PRINTLOGLENGTH      = 40
EXIT_GOOD           = 0
EXIT_FAIL           = 1
SORT_MODES          = ['hostname', 'IP', 'first_seen', 'expiry', 'MAC', 'MACOUI', 'notes']
DEFAULT_SORT_MODE   = "hostname"   # If not specified in config file or command line
PY_VERSION          = sys.version_info
# SYSTEM              = platform.system()     # 'Linux', 'Windows', ...

def main():

    logging.getLogger().setLevel(20)  # Set INFO level for interactive use
    db_connect()

    # List known clients from database
    if args.list_db or args.SearchTerm != None:
        get_database_clients(dump=True, search=args.SearchTerm)
        cleanup(EXIT_GOOD)

    # List known clients from dhcp server
    if args.list_dhcp_server:
        get_dhcp_clients(dump=True)
        cleanup(EXIT_GOOD)

    # Add a note for a client to the database
    if args.note is not None:
        if args.MAC is None:
            logging.error ("Must specify the --MAC when using --add-note")
            cleanup(EXIT_FAIL)
        clients = db_cursor.execute(f"SELECT * FROM {getcfg('DB_TABLE')} WHERE MAC = '{args.MAC}'").fetchall()
        if len(clients) == 1:
            new_note = args.note.replace("'", "''")   # single quotes need to be doubled up in SQL
            clients = get_database_clients()
            logging.info (f"Note added for {args.MAC} / {clients[args.MAC]['hostname']}:  {new_note}")
            db_cursor.execute(f"UPDATE {getcfg('DB_TABLE')} SET notes = '{new_note}' WHERE MAC = '{args.MAC}'")
            db_connection.commit()
            cleanup(EXIT_GOOD)
        elif len(clients) > 1:
            logging.error (f"MAC address <{args.MAC}> found more than once in the database. Aborting.")
            cleanup(EXIT_FAIL)
        else:
            logging.error (f"MAC address <{args.MAC}> not found in the database. Aborting.")
            cleanup(EXIT_FAIL)

    # Delete a client from the database
    if args.delete:
        if args.MAC is None:
            logging.error ("ERROR - Must specify the --MAC when using --delete")
            cleanup(EXIT_FAIL)
        clients = db_cursor.execute(f"SELECT * FROM {getcfg('DB_TABLE')} WHERE MAC = '{args.MAC}'").fetchall()
        if len(clients) == 1:
            clients = get_database_clients()
            logging.info (f"Deleted {args.MAC} / {clients[args.MAC]['hostname']}")
            db_cursor.execute(f"DELETE FROM {getcfg('DB_TABLE')} WHERE MAC = '{args.MAC}'")
            db_connection.commit()
            cleanup(EXIT_GOOD)
        elif len(clients) > 1:
            logging.error (f"MAC address <{args.MAC}> found more than once in the database. Aborting.")
            cleanup(EXIT_FAIL)
        else:
            logging.error (f"MAC address <{args.MAC}> not found in the database. Aborting.")
            cleanup(EXIT_FAIL)

    # Update/refresh the database from the dhcp server
    if args.update:
        do_update()
        cleanup(EXIT_GOOD)

    logging.error ("Nothing to do.  Use one of --list-db, --list-dhcp-server, --update, --add-note, --delete, or --create-db.")
    cleanup(EXIT_FAIL)

def service():
    db_connect()
    next_run = time.time()
    while True:
        if config.loadconfig(call_logfile_wins=logfile_override, flush_on_reload = True):
            logging.warning(f"NOTE - The config file has been reloaded.")
            next_run = time.time()                  # Force immediate update if the config file is touched

        if time.time() > next_run:
            do_update()
            next_run += timevalue(getcfg("UpdateInterval")).seconds
        time.sleep(5)

def db_connect():
    """ Check for database access and populate if not existing """
    global db_connection, db_cursor

    mungePath(tool.data_dir, mkdir=True)    # Force make the data_dir
    _fp = str(mungePath(getcfg("DB_DB"), tool.data_dir).full_path)
    db_connection = sqlite3.connect(_fp)    # _fp must be str on Python 3.6.8
    db_connection.row_factory = sqlite3.Row
    db_cursor = db_connection.cursor()
    found = False
    for row in db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall():
        if getcfg('DB_TABLE') in row['name']:
            found = True
            break

    if not found:
        logging.warning ("Network clients database table not found.")

    if found and args.create_db:
        logging.warning ("Deleted existing network clients database table.")
        db_cursor.execute(f"DROP TABLE {getcfg('DB_TABLE')}")
        db_connection.commit()

    db_fields = "MAC VARCHAR(17), hostname VARCHAR(25), notes VARCHAR(255), first_seen INT, expiry INT, IP VARCHAR(15), MACOUI VARCHAR(255)"
    if not found or args.create_db:
        logging.warning ("Creating network clients database table:")
        dhcp_clients = get_dhcp_clients()
        db_cursor.execute(f"CREATE TABLE {getcfg('DB_TABLE')} ({db_fields})")

        count = 0
        for MAC in dhcp_clients:
            count += 1
            macoui = db_add_client (MAC, dhcp_clients[MAC])
            logging.info (f"  {dhcp_clients[MAC]['hostname']:25}  {dhcp_clients[MAC]['IP']:15}  {MAC}   {macoui}")
        db_connection.commit()
        logging.info  (f"Database table created with  <{count}>  clients.")
        cleanup(EXIT_GOOD)

def do_update():
    """ Update database from dhcp server
    """
    try:
        dhcp_clients = get_dhcp_clients()
        database_clients = get_database_clients()
        for MAC in dhcp_clients:
            if MAC not in database_clients:
                macoui = db_add_client (MAC, dhcp_clients[MAC])
                subject = "New LAN client found"
                msg = f"\n  Hostname:    {dhcp_clients[MAC]['hostname']}\
                        \n  IP address:  {dhcp_clients[MAC]['IP']}\
                        \n  MAC:         {MAC}\
                        \n  MACOUI:      {macoui}"
                try:
                    snd_notif (subj=subject, msg=msg, log=True)
                except Exception as e:
                    logging.warning(f"snd_notif error for <{subject}>:  {e}")
                continue
            if dhcp_clients[MAC]['hostname'] != database_clients[MAC]['hostname']:
                msg = (f"{MAC} / {database_clients[MAC]['hostname']:<20} New hostname: {dhcp_clients[MAC]['hostname']}")
                logging.info(msg)
                db_cursor.execute(f"UPDATE {getcfg('DB_TABLE')} SET hostname = \'{dhcp_clients[MAC]['hostname']}\' WHERE MAC = '{MAC}'")
            if dhcp_clients[MAC]['IP'] != database_clients[MAC]['IP']:
                msg = (f"{MAC} / {database_clients[MAC]['hostname']:<20} New IP:       {dhcp_clients[MAC]['IP']}")
                logging.info(msg)
                db_cursor.execute(f"UPDATE {getcfg('DB_TABLE')} SET IP = \'{dhcp_clients[MAC]['IP']}\' WHERE MAC = '{MAC}'")
            if dhcp_clients[MAC]['expiry'] != database_clients[MAC]['expiry']:
                expiry = datetime.datetime.fromtimestamp(dhcp_clients[MAC]['expiry'])
                msg = (f"{MAC} / {database_clients[MAC]['hostname']:<20} New expiry:   {expiry}")
                logging.info(msg)
                db_cursor.execute(f"UPDATE {getcfg('DB_TABLE')} SET expiry = \'{dhcp_clients[MAC]['expiry']}\' WHERE MAC = '{MAC}'")

        db_connection.commit()
    except Exception as e:
        logging.warning(f"Failed in do_update:\n{e}")

def db_add_client(MAC, record):
    """ Add a single client to the database.
    Returns macoui so that do_update may use if for notification
    """
    macoui = lookup_MAC(MAC)
    cmd = "INSERT INTO {} (MAC, MACOUI, hostname, notes, first_seen, IP, expiry) VALUES (\'{}\', \'{}\', \'{}\', \'{}\', {}, \'{}\', {})".format(
        getcfg('DB_TABLE'),
        MAC,
        macoui,
        record['hostname'],
        "-",
        time.time(),
        record['IP'],
        record['expiry'])
    db_cursor.execute(cmd)
    return macoui

next_lookup = time.time()
def lookup_MAC(MAC):
    """ Given MAC 00:05:cd:8a:22:33, get OUI from https://api.macvendors.com/00:05:cd
    Lookups limited to 2 per seconds without a macvendors.com account
    """
    global next_lookup
    while 1:
        if time.time() > next_lookup:
            break
    macoui = "--none--"
    r = requests.get("https://api.macvendors.com/" + MAC[0:8])
    if r.status_code == 200:
        macoui = r.text
    next_lookup = time.time() + 0.6
    return macoui

    # Remnant implementation using https://oidsearch.s.dd-wrt.com
    # # Given MAC 00:05:cd:8a:22:33, web page returns
    # #
Result for OID 00:05:cd

    # #   Denon, Ltd.

35-1, Sagamiono 7-chome,

    # #   Kanagawa    228-8505

    # # lookup_MAC() returns "Denon, Ltd."
    # r = requests.get("https://oidsearch.s.dd-wrt.com/search/" + MAC[0:8])
    # if r.status_code == 200:
    #     _next = False
    #     for line in r.text.split("\n"):
    #         if _next:
    #             break
    #         if "
" in line:
    #             _next = True
    #     if _next:
    #         if "
" in line:
    #             search_result = line[0:line.find("
")]
    #         else:
    #             search_result = line
    #         if search_result == "":
    #             search_result = "--none--"
    # else:
    #     search_result = "OID search failed"

    # # return (search_result)
    # return html.unescape(search_result)

def get_database_clients(dump=False, search=None):
    """ Get clients currently in the database, return a dictionary of dictionaries, keyed by MAC
        {
            MAC:  { hostname:, IP:, expiry:, first_seen:, notes:, MACOUI: }
        }
    """
    global sort_by

    clients_list = {}
    for row in db_cursor.execute(f"SELECT * FROM {getcfg('DB_TABLE')}").fetchall():
        clients_list[row['MAC']] = {
            "hostname":   row['hostname'],
            "IP":         row['IP'],
            "expiry":     row['expiry'],
            "first_seen": row['first_seen'],
            "notes":      row['notes'],
            "MACOUI":     row['MACOUI'] }

    if dump:
        if sort_by == 'MAC':
            clients_list = collections.OrderedDict(sorted(clients_list.items()))
        elif sort_by in ["first_seen", "expiry"]:
            clients_list = collections.OrderedDict(sorted(clients_list.items(), key=lambda t:t[1][sort_by]))
        else:
            clients_list = collections.OrderedDict(sorted(clients_list.items(), key=lambda t:t[1][sort_by].lower()))
        first=True
        if search == None:
            search = ""
        search = search.lower()
        count = 0
        for MAC in clients_list:

            first_seen = str(datetime.datetime.fromtimestamp(int(clients_list[MAC]['first_seen'])))
            if clients_list[MAC]['expiry'] == 0:
                expiry = "static lease"
            else:
                expiry = str(datetime.datetime.fromtimestamp(int(clients_list[MAC]['expiry'])))

            if search=="" or \
                        search in MAC or \
                        search in clients_list[MAC]['hostname'].lower() or \
                        search in clients_list[MAC]['IP'] or \
                        search in expiry.lower() or \
                        search in first_seen.lower() or \
                        search in clients_list[MAC]['MACOUI'].lower() or \
                        search in clients_list[MAC]['notes'].lower():
                count += 1
                if first:
                    print(f"{'Hostname':<25}  {'First seen':<19}  {'Current IP':<15}  {'IP Expiry':<19}  {'MAC':<17}  "
                        + f"{'MAC Org Unique ID':<{getcfg('MACOUI_field_width', 30)}}  {'Notes'}")
                    first = False
                _macoui = clients_list[MAC]['MACOUI'][:getcfg('MACOUI_field_width', 30)]
                print(f"{clients_list[MAC]['hostname']:<25}  {first_seen:<19}  {clients_list[MAC]['IP']:<15}  {expiry:<19}  {MAC:<17}  "
                    + f"{_macoui:<{getcfg('MACOUI_field_width', 30)}}  {clients_list[MAC]['notes']}")
        print (f"  <{count}>  known clients.")

    return clients_list

def get_dhcp_clients(dump=False):
    """ Get leases from the dhcp server, return a sorted dictionary of dictionaries, keyed by MAC
        {
            MAC:  { hostname:, IP:, expiry: }
        }
    """
    clients_list = {}
    clients = ""
    if getcfg("DHCP_SERVER_TYPE").lower() == "pfsense":
        pf_clients = scrape_pfsense_dhcp(getcfg("PF_DHCP_URL"), getcfg("PF_USER"), getcfg("PF_PASS"))
        for row in pf_clients:
            if row["End"] == "n/a":
                expiry_timestamp = 0
            else:
                expiry_timestamp = datetime.datetime.strptime(row["End"], getcfg("PF_DATE_FORMAT")).timestamp()    # '2021/11/07 11:51:44'
            clients_list[row["MAC address"]] = {"hostname":row["Hostname"], "IP":row["IP address"], "expiry":expiry_timestamp}

    elif getcfg("DHCP_SERVER_TYPE").lower() == "dd-wrt":
        try:
            _cmd = ["ssh", "root@" + getcfg("DDWRT_IP"), "-o", "ConnectTimeout=1", "-T", "cat", getcfg("DDWRT_DHCP")]
            if PY_VERSION >= (3, 7): #3.7:
                clients = subprocess.run(_cmd, capture_output=True, text=True).stdout.split("\n")
            else:   #Py 3.6 .run does not support capture_output, so use the old method.
                clients = subprocess.run(_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout.split("\n")
        except Exception as e:
            logging.warning("Exception <{}>".format(e))
        # 1587457675 00:0d:c5:5c:82:6d 192.168.1.105 Hopper-ETH0 01:00:0d:c5:5c:82:6d
        line_re = re.compile(r"([\d]+) ([\dabcdef:]+) ([\d.]+) ([\S]+)") # [\dabcdef:]+")
        for line in clients:
            xx = line_re.match(line)
            if xx:
                clients_list[xx.group(2)] = {"hostname":xx.group(4), "IP":xx.group(3), "expiry":int(xx.group(1))}
            else:
                if len(line) > 0:
                    logging.warning ("ERROR in get_dhcp_clients:  This line looks bogus:\n  ", line)
    else:
        logging.error (f"Invalid DHCP_SERVER_TYPE in config: {getcfg('DHCP_SERVER_TYPE')}")
        cleanup(EXIT_FAIL)

    if dump:
        if sort_by == 'MAC':
            clients_list = collections.OrderedDict(sorted(clients_list.items()))
        elif sort_by in ["hostname", "IP"]:
            clients_list = collections.OrderedDict(sorted(clients_list.items(), key=lambda t:t[1][sort_by].lower()))
        elif sort_by == "expiry":
            clients_list = collections.OrderedDict(sorted(clients_list.items(), key=lambda t:t[1][sort_by]))
        else:
            print (f"For --list-dhcp-server, <--sort-by {sort_by}> not supported.  Defaulting to <--sort-by hostname>.")
            clients_list = collections.OrderedDict(sorted(clients_list.items(), key=lambda t:t[1]["hostname"].lower()))

        count = 0
        for client in clients_list:
            count += 1
            if clients_list[client]["expiry"] == 0:
                expiry = "static lease"
            else:
                expiry = str(datetime.datetime.fromtimestamp(int(clients_list[client]["expiry"])))

            print(f"{expiry:<19}  {client}  {clients_list[client]['IP']:<15}  {clients_list[client]['hostname']}")
        print (f"  <{count}>  known clients.")

    return clients_list

def scrape_pfsense_dhcp(url, user, password):
    """Queries the pfsense (+v2.4) dhcp leases status page and returns a list of dictionaries, one for each table row.
    The dictionary keys are the column names, such as "Hostname", "IP address", and "MAC address".

    Adapted from Github pletch/scrape_pfsense_dhcp_leases.py (https://gist.github.com/pletch/037a4a01c95688fff65752379534455f), thank you pletch.
    """
    s = requests.session()
    try:
        r = s.get(url,verify = False)
    except:
        logging.error(f"Error attempting to connect to <{url}>.  url and login credentials valid?")
        sys.exit(EXIT_FAIL)

    matchme = 'csrfMagicToken = "(.*)";var'
    csrf = re.search(matchme,str(r.text))
    payload = {'__csrf_magic' : csrf.group(1), 'login' : 'Login', 'usernamefld' : user, 'passwordfld' : password}
    r = s.post(url,data=payload,verify = False)
    r = s.get(url,verify = False)
    tree = _html.fromstring(r.content)

    headers = []
    none_index = 0
    tr_elements = tree.xpath('//tr')
    for header in tr_elements[0]:
        name = header.text
        if name == None:            # Ensure unique name for each column with no name
            name = "None" + str(none_index)
            none_index += 1
        headers.append(name)

    least_list = []
    xpath_base = '//body[1]//div[1]//div[2]//div[2]//table[1]//tbody//tr'
    for row in tree.xpath(xpath_base):
        row_dict = {}
        header_index = 0
        for node in row:
            item_text = node.text
            if item_text != None:
                item_text = item_text.strip()
            row_dict[headers[header_index]] = item_text
            header_index += 1
        least_list.append(row_dict)

    return(least_list)

def cleanup(exit_code):
    if exit_code == EXIT_FAIL:
        logging.warning (f"Cleanup and exit({exit_code})")
    if db_cursor:
        db_cursor.close()
    if db_connection:
        db_connection.close()
    sys.exit(exit_code)
    pass

def int_handler(signal, frame):
    logging.warning(f"Signal {signal} received.  Exiting.")
    cleanup(EXIT_FAIL)

signal.signal(signal.SIGINT,  int_handler)      # Ctrl-C
signal.signal(signal.SIGTERM, int_handler)      # kill

def cli():
    global tool, config, args, logfile_override
    global sort_by

    tool = set_toolname (TOOLNAME)
    parser = argparse.ArgumentParser(description=__doc__ + __version__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('SearchTerm', nargs='?',
                        help="Print database records containing this text.")
    parser.add_argument('--update', '-u', action="store_true",
                        help="Check the dhcp server for new connections and update the database.")
    parser.add_argument('--list-db', '-l', action='store_true',
                        help="Print known clients on the network from the database (default mode).")
    parser.add_argument('--list-dhcp-server', '-r', action='store_true',
                        help="Print known clients on the network from the dhcp server.")
    parser.add_argument('--sort-by', '-s', choices=SORT_MODES,
                        help=f"Sort --list-db and --list-dhcp-server output. Overrides config SortBy. Default <{DEFAULT_SORT_MODE}> if neither specified.")
    parser.add_argument('--create-db', action='store_true',
                        help="Create a fresh database and populate it with the current dhcp server clients.")
    parser.add_argument('--note', '-n', type=str,
                        help="Add a note to the db for the specified --MAC.")
    parser.add_argument('--delete', action='store_true',
                        help="Delete from the db the specified --MAC.")
    parser.add_argument('--MAC', '-m', type=str,
                        help="MAC address for --add-note or --delete.")
    parser.add_argument('--config-file', '-c', type=str, default=CONFIG_FILE,
                        help=f"Path to the config file (Default <{CONFIG_FILE})> in user/site config directory.")
    parser.add_argument('--print-log', '-p', action='store_true',
                        help=f"Print the tail end of the log file (default last {PRINTLOGLENGTH} lines).")
    parser.add_argument('--service', action='store_true',
                        help="Run updates in an endless loop for use as a systemd service.")
    parser.add_argument('--setup-user', action='store_true',
                        help=f"Install starter files in user space.")
    parser.add_argument('--setup-site', action='store_true',
                        help=f"Install starter files in system-wide space. Run with root prev.")
    parser.add_argument('-V', '--version', action='version', version=f"{tool.toolname} {__version__}",
                        help="Return version number and exit.")
    args = parser.parse_args()

    # Deploy template files
    if args.setup_user:
        deploy_files([
            { "source": CONFIG_FILE,             "target_dir": "USER_CONFIG_DIR", "file_stat": 0o644, "dir_stat": 0o755},
            { "source": "creds_SMTP",            "target_dir": "USER_CONFIG_DIR", "file_stat": 0o600},
            { "source": "creds_routermonitor",   "target_dir": "USER_CONFIG_DIR", "file_stat": 0o600},
            { "source": "routermonitor.service", "target_dir": "USER_CONFIG_DIR", "file_stat": 0o644},
            ]) #, overwrite=True)
        sys.exit()

    if args.setup_site:
        deploy_files([
            { "source": CONFIG_FILE,             "target_dir": "SITE_CONFIG_DIR", "file_stat": 0o644, "dir_stat": 0o755},
            { "source": "creds_SMTP",            "target_dir": "SITE_CONFIG_DIR", "file_stat": 0o600},
            { "source": "creds_routermonitor",   "target_dir": "SITE_CONFIG_DIR", "file_stat": 0o600},
            { "source": "routermonitor.service", "target_dir": "SITE_CONFIG_DIR", "file_stat": 0o644},
            ]) #, overwrite=True)
        sys.exit()

    # Load config file and setup logging
    logfile_override = True  if not args.service  else False
    try:
        config = config_item(args.config_file)
        config.loadconfig(call_logfile_wins=logfile_override)         #, call_logfile=args.log_file, ldcfg_ll=10)
    except Exception as e:
        logging.error(f"Failed loading config file <{args.config_file}>. \
\n  Run with  '--setup-user' or '--setup-site' to install starter files.\n  {e}\n  Aborting.")
        sys.exit(EXIT_FAIL)

    logging.warning (f"========== {tool.toolname} ({__version__}) ==========")
    logging.warning (f"Config file <{config.config_full_path}>")

    # Print log
    if args.print_log:
        try:
            _lf = mungePath(getcfg("LogFile"), tool.log_dir_base).full_path
            print (f"Tail of  <{_lf}>:")
            _xx = collections.deque(_lf.open(), getcfg("PrintLogLength", PRINTLOGLENGTH))
            for line in _xx:
                print (line, end="")
        except Exception as e:
            print (f"Couldn't print the log file.  LogFile defined in the config file?\n  {e}")
        sys.exit()

    sort_by = getcfg("SortBy", DEFAULT_SORT_MODE)
    if args.sort_by:
        sort_by = args.sort_by
    if sort_by not in SORT_MODES:
        logging.error(f"Invalid SortBy value <{getcfg('SortBy')}> in config file")
        sys.exit(EXIT_FAIL)

    if not args.update and not args.list_dhcp_server and not args.print_log and not args.create_db and not args.note and not args.delete and not args.service:
        args.list_db = True     # If operation/mode not specified then default to list_db

    if args.service:
        service()

    sys.exit(main())

if __name__ == '__main__':
    sys.exit(cli())


import fnmatch
import os
import warnings
from concurrent.futures import ProcessPoolExecutor, as_completed
from functools import partial
from pprint import pprint

import exiftool
from tqdm import tqdm

import micasense.capture as capture
import micasense.image as image

warnings.simplefilter(action="once")

# FIXME: mirrors Capture.append_file(). Not used. Does this still belong here?
def image_from_file(filename):
    return image.Image(filename)

def parallel_process(function, iterable, parameters, progress_callback=None, use_tqdm=False):
    """
    Multiprocessing Pool handler.
    :param function: function used in multiprocessing call
    :param iterable: iterable holding objects passed to function for each process
    :param parameters: dict of any function parameters other than the iterable object
    :param use_tqdm: boolean True to use tqdm progress bar
    :param progress_callback: function to report progress to
    :return: None
    """

    with ProcessPoolExecutor() as pool:
        # run multiprocessing
        futures = [pool.submit(partial(function, parameters), i) for i in iterable]

        if use_tqdm:
            # kwargs for tqdm
            kwargs = {
                'total': len(futures),
                'unit': 'Capture',
                'unit_scale': False,
                'leave': True
            }

            # Receive Future objects as they complete. Print out the progress as tasks complete
            for _ in tqdm(iterable=as_completed(futures), desc='Processing ImageSet', **kwargs):
                pass
        elif progress_callback is not None:
            futures_len = float(len(futures))
            for i, _ in enumerate(as_completed(futures)):
                progress_callback(float(i) / futures_len)

def save_capture(params, cap):
    """
    Process an ImageSet according to program parameters. Saves rgb
    :param params: dict of program parameters from ImageSet.process_imageset()
    :param cap: micasense.capture.Capture object
    """
    try:
        # align capture
        if len(cap.images) == params['capture_len']:
            cap.create_aligned_capture(
                irradiance_list=params['irradiance'],
                warp_matrices=params['warp_matrices'],
                img_type=params['img_type']
            )
        else:
            print(f"\tCapture {cap.uuid} only has {len(cap.images)} Images. Should have {params['capture_len']}. "
                  f"Skipping...")
            return

        if params['output_stack_dir']:
            output_stack_file_path = os.path.join(params['output_stack_dir'], cap.uuid + '.tif')
            if params['overwrite'] or not os.path.exists(output_stack_file_path):
                cap.save_capture_as_stack(output_stack_file_path)
        if params['output_rgb_dir']:
            output_rgb_file_path = os.path.join(params['output_rgb_dir'], cap.uuid + '.jpg')
            if params['overwrite'] or not os.path.exists(output_rgb_file_path):
                cap.save_capture_as_rgb(output_rgb_file_path)

        cap.clear_image_data()
    except Exception as e:
        print(e)
        pprint(params)
        quit()

class ImageSet(object):
    """An ImageSet is a container for a group of Captures that are processed together."""

    def __init__(self, captures):
        self.captures = captures
        captures.sort()

    @classmethod
    def from_directory(cls, directory, progress_callback=None, use_tqdm=False, exiftool_path=None):
        """
        Create an ImageSet recursively from the files in a directory.
        :param directory: str system file path
        :param progress_callback: function to report progress to
        :param use_tqdm: boolean True to use tqdm progress bar
        :param exiftool_path: str system file path to exiftool location
        :return: ImageSet instance
        """

        # progress_callback deprecation warning
        if progress_callback is not None:
            warnings.warn(message='The progress_callback parameter will be deprecated in favor of use_tqdm',
                          category=PendingDeprecationWarning)

        # ensure exiftoolpath is found per MicaSense setup instructions
        if exiftool_path is None and os.environ.get('exiftoolpath') is not None:
            exiftool_path = os.path.normpath(os.environ.get('exiftoolpath'))

        cls.basedir = directory
        matches = []
        for root, _, filenames in os.walk(directory):
            [matches.append(os.path.join(root, filename)) for filename in fnmatch.filter(filenames, '*.tif')]

        images = []

        with exiftool.ExifTool(exiftool_path) as exift:
            if use_tqdm:  # to use tqdm progress bar instead of progress_callback
                kwargs = {
                    'total': len(matches),
                    'unit': ' Files',
                    'unit_scale': False,
                    'leave': True
                }
                for path in tqdm(iterable=matches, desc='Loading ImageSet', **kwargs):
                    images.append(image.Image(path, exiftool_obj=exift))
            else:
                print('Loading ImageSet from: {}'.format(directory))
                for i, path in enumerate(matches):
                    images.append(image.Image(path, exiftool_obj=exift))
                    if progress_callback is not None:
                        progress_callback(float(i) / float(len(matches)))

        # create a dictionary to index the images so we can sort them into captures
        # {
        #     "capture_id": [img1, img2, ...]
        # }
        captures_index = {}
        for img in images:
            c = captures_index.get(img.capture_id)
            if c is not None:
                c.append(img)
            else:
                captures_index[img.capture_id] = [img]
        captures = []
        for cap_imgs in captures_index:
            imgs = captures_index[cap_imgs]
            newcap = capture.Capture(imgs)
            captures.append(newcap)
        if progress_callback is not None:
            progress_callback(1.0)
        return cls(captures)

    def as_nested_lists(self):
        """
        Get timestamp, latitude, longitude, altitude, capture_id, dls-yaw, dls-pitch, dls-roll, and irradiance from all
        Captures.
        :return: List data from all Captures, List column headers.
        """
        columns = [
            'timestamp',
            'latitude', 'longitude', 'altitude',
            'capture_id',
            'dls-yaw', 'dls-pitch', 'dls-roll'
        ]
        irr = ["irr-{}".format(wve) for wve in self.captures[0].center_wavelengths()]
        columns += irr
        data = []
        for cap in self.captures:
            dat = cap.utc_time()
            loc = list(cap.location())
            uuid = cap.uuid
            dls_pose = list(cap.dls_pose())
            irr = cap.dls_irradiance()
            row = [dat] + loc + [uuid] + dls_pose + irr
            data.append(row)
        return data, columns

    def dls_irradiance(self):
        """
        Get utc_time and irradiance for each Capture in ImageSet.
        :return: dict {utc_time : [irradiance, ...]}
        """
        series = {}
        for cap in self.captures:
            dat = cap.utc_time().isoformat()
            irr = cap.dls_irradiance()
            series[dat] = irr
        return series

    def process_imageset(self,
                         output_stack_directory=None,
                         output_rgb_directory=None,
                         warp_matrices=None,
                         irradiance=None,
                         img_type=None,
                         multiprocess=True,
                         overwrite=False,
                         progress_callback=None,
                         use_tqdm=False):
        """
        Write band stacks and rgb thumbnails to disk.
        :param warp_matrices: 2d List of warp matrices derived from Capture.get_warp_matrices()
        :param output_stack_directory: str system file path to output stack directory
        :param output_rgb_directory: str system file path to output thumbnail directory
        :param irradiance: List returned from Capture.dls_irradiance() or Capture.panel_irradiance()    <-- TODO: Write a better docstring for this
        :param img_type: str 'radiance' or 'reflectance'. Desired image output type.
        :param multiprocess: boolean True to use multiprocessing module
        :param overwrite: boolean True to overwrite existing files
        :param progress_callback: function to report progress to
        :param use_tqdm: boolean True to use tqdm progress bar
        """

        if progress_callback is not None:
            warnings.warn(message='The progress_callback parameter will be deprecated in favor of use_tqdm',
                          category=PendingDeprecationWarning)

        # ensure some output is requested
        if output_stack_directory is None and output_rgb_directory is None:
            raise RuntimeError('No output requested for the ImageSet.')

        # make output dirs if not exist
        if output_stack_directory is not None and not os.path.exists(output_stack_directory):
            os.mkdir(output_stack_directory)
        if output_rgb_directory is not None and not os.path.exists(output_rgb_directory):
            os.mkdir(output_rgb_directory)

        # processing parameters
        params = {
            'warp_matrices': warp_matrices,
            'irradiance': irradiance,
            'img_type': img_type,
            'capture_len': len(self.captures[0].images),
            'output_stack_dir': output_stack_directory,
            'output_rgb_dir': output_rgb_directory,
            'overwrite': overwrite,
        }

        print('Processing {} Captures ...'.format(len(self.captures)))

        # multiprocessing with concurrent futures
        if multiprocess:
            parallel_process(function=save_capture, iterable=self.captures, parameters=params,
                             progress_callback=progress_callback, use_tqdm=use_tqdm)

        # else serial processing
        else:
            if use_tqdm:
                kwargs = {
                    'total': len(self.captures),
                    'unit': 'Capture',
                    'unit_scale': False,
                    'leave': True
                }
                for cap in tqdm(iterable=self.captures, desc='Processing ImageSet', **kwargs):
                    save_capture(params, cap)
            else:
                for i, cap in enumerate(self.captures):
                    save_capture(params, cap)
                    if progress_callback is not None:
                        progress_callback(float(i) / float(len(self.captures)))

        print('Processing complete.')


import sys
import re

from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import OocLexer

pattern = re.compile(
    r'(~{3,})(.+?)\1',
    re.S
)

formatter = HtmlFormatter( style='trac' )
lexer     = OocLexer()

sourceFile = open( sys.argv[ 1 ], 'r+' )
source = sourceFile.read()

matches = pattern.finditer( source )

for match in matches:
    source = source.replace( match.group(), highlight( match.group( 2 ), lexer, formatter ) )

sourceFile.seek( 0 )
sourceFile.write( source )
sourceFile.close()

css = formatter.get_style_defs( '.highlight' )
cssFile = open( sys.argv[ 2 ] + '/highlight.css', 'w' )
cssFile.write( '\n/* Pygments CSS for style "trac" */\n\n' )
cssFile.write( css )
cssFile.close()


# -*- coding: utf-8 -*-
"""
Created on Wed Jul  8 22:06:23 2020

@author: Lou
"""

import sys, yaml, os
#os.chdir('C:/Users/Keith/Documents/GitHub/LNT')
## with open('.' + os.sep + 'loc_settings.yaml', 'r') as f:
##     loc_info = yaml.load(f)
#sys.path.append('C:/Users/Keith/Documents/GitHub/LNT'+ os.sep + "Analysis")
#sys.path.append('C:/Users/Keith/Documents/GitHub/LNT'+ os.sep + "Imaging")
#sys.path.append('C:/Users/Keith/Documents/GitHub/OASIS-master')

os.chdir(r'C:\Users\Lou\Documents\repos\LNT')
sys.path.append(r'C:\Users\Lou\Documents\repos\LNT'+ os.sep + "Analysis")
sys.path.append(r'C:\Users\Lou\Documents\repos\LNT'+ os.sep + "Imaging")
sys.path.append(r'C:\Users\Lou\Documents\repos\OASIS-master')

import warnings; warnings.simplefilter('ignore')

import matplotlib.pyplot as plt
import h5py
import numpy as np
from scipy import stats
from align_dF_interp_mpi import align_dF
from load_behavior_data import load_data
from oasis.functions import deconvolve
from sklearn import linear_model

def process_and_align_caiman(data_path, sess, dff_file, behavior_file):
    eye_data = None
    session_crop=[0,1]

    print('Loading df/f data...')
    df_signal_path = data_path + os.sep + dff_file
    dF_signal_bef =  h5py.File(df_signal_path, 'r+')

    try:
        del dF_signal_bef['dF_aligned']
    except KeyError:
        pass
    try:
        del dF_signal_bef['behaviour_aligned']
    except KeyError:
        pass

    dF_signal = np.copy(dF_signal_bef['F_dff'])
    frame_brightness = np.zeros(len(dF_signal))

    print('Loading behavior data for alignment...')
    fname = data_path + os.sep + behavior_file
    raw_data = np.genfromtxt(fname, delimiter=';')

    dF_aligned, behaviour_aligned, bri_aligned = align_dF(raw_data, dF_signal, frame_brightness,[-1, -3], [2, -4], True, True, session_crop, eye_data)

    # del dF_signal_bef['dF_aligned']
    dF_signal_bef.create_dataset('dF_aligned', data=dF_aligned)
    # del dF_signal_bef['behaviour_aligned']
    dF_signal_bef.create_dataset('behaviour_aligned', data=behaviour_aligned)
    dF_signal_bef.close()

    print('done...')

    return dF_aligned, behaviour_aligned

def factor_out_baseline(data_path, sess, dff_file):

    print('Loading df/f data...')
    df_signal_path = data_path + os.sep + dff_file
    dF_signal_bef =  h5py.File(df_signal_path, 'r+')
    try:
        del dF_signal_bef['new_dF']
    except KeyError:
        pass
    try:
        del dF_signal_bef['calcium_dF']
    except KeyError:
        pass
    dF_signal = np.copy(dF_signal_bef['dF_aligned'])
    new_dF = np.zeros((len(dF_signal[:,0]),len(dF_signal[0,:])))
    calcium_dF = np.zeros((len(dF_signal[:,0]),len(dF_signal[0,:])))
    print('Calculate baselines.')
    for num in range(len(dF_signal[0,:])):
        test_sig = dF_signal[:,num]
        rep_sig = np.zeros(len(test_sig))
        ker_leg = 1000
        x_up_lim = len(test_sig) - ker_leg
        for x in range(len(test_sig)):
            if x < ker_leg:
                cur_vals = test_sig[:x + ker_leg]
            elif x > x_up_lim:
                cur_vals = test_sig[x-ker_leg:]
            else:
                cur_vals = test_sig[x-ker_leg:x+ker_leg]
            sort_val = np.sort(cur_vals)
            bottom_20 = int(len(cur_vals)*0.2)
            cur_val = np.sum(sort_val[:bottom_20])
            rep_sig[x] = cur_val/bottom_20
        current_dF = test_sig-rep_sig

        xx = np.fft.fft(current_dF)
        xx[0] = 0
        yy = np.fft.ifft(xx)
        final_dF = yy.astype('float64')
        new_dF[:,num] = final_dF

        ransac = linear_model.RANSACRegressor()
        ransac.fit(np.arange(len(current_dF)).reshape(-1, 1), current_dF.reshape(-1, 1))
        prediction_ransac = ransac.predict(np.arange(len(current_dF)).reshape(-1, 1))
        calcium_dF[:,num] = current_dF-prediction_ransac[int(len(current_dF)/2),0]

    dF_signal_bef.create_dataset('new_dF', data=new_dF)
    dF_signal_bef.create_dataset('calcium_dF', data=calcium_dF)
    dF_signal_bef.close()

    print('done...')

    return None

def calculate_detrend_and_deconvolve(data_path, sess, dff_file):

    print('Loading df/f data...')
    df_signal_path = data_path + os.sep + dff_file
    dF_signal_bef =  h5py.File(df_signal_path, 'r+')
    dF_signal = np.copy(dF_signal_bef['new_dF'])
    try:
        del dF_signal_bef['deconv']
    except KeyError:
        pass
    try:
        del dF_signal_bef['spikes']
    except KeyError:
        pass
    deconv = np.zeros((len(dF_signal[:,0]),len(dF_signal[0,:])))
    spikes = np.zeros((len(dF_signal[:,0]),len(dF_signal[0,:])))

    print('Calculate detrends and convolutions.')
    for num in range(len(dF_signal[0,:])):
        current_dF = dF_signal[:,num]
        c, s, b, g, lam = deconvolve(current_dF, penalty=0)
        deconv[:,num] = c
        spikes[:,num] = s

    dF_signal_bef.create_dataset('deconv', data=deconv)
    dF_signal_bef.create_dataset('spikes', data=spikes)
    dF_signal_bef.close()
    print('done...')

    return None

def run_test():
    MOUSE= 'LF191022_1'
    sess = '20191209'
    dff_file = 'M01_000_000_results.mat'
    data_path = 'Q:\Documents\Harnett UROP' + os.sep + MOUSE + os.sep + sess
    factor_out_baseline(data_path, sess, dff_file)

if __name__ == '__main__':
    run_test()

        # ransac = linear_model.RANSACRegressor()
        # ransac.fit(np.arange(len(current_dF)).reshape(-1, 1), current_dF.reshape(-1, 1))
        # offset_predict = ransac.predict(np.arange(len(current_dF)).reshape(-1, 1))


from __future__ import print_function
import sys
import copy
import logging
import itertools
import time
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from sklearn.metrics import precision_recall_curve, average_precision_score, roc_auc_score
from sklearn import metrics
from scipy.stats import pearsonr, spearmanr
import numpy as np
from fingerprint.graph import load_from_mol_tuple
from utils import get_lstm_embedding

class Evaluator():
    def __init__(self, ikey2smiles=None,
                       ikey2mol=None,
                       berttokenizer=None,
                       uniprot2triplets=None,
                       prediction_mode=None,
                       protein_embedding_type=None,
                       datatype='train',
                       max_steps=1000,
                       batch=64,
                       shuffle=False):

        self.ikey2smiles=ikey2smiles
        self.ikey2mol=ikey2mol
        self.berttokenizer=berttokenizer
        self.uniprot2triplets=uniprot2triplets
        self.prediction_mode=prediction_mode
        self.prottype=protein_embedding_type
        self.datatype=datatype
        self.max_steps=max_steps
        self.batch=batch
        self.shuffle=shuffle
        logging.info("{} Evaluator for {} data initialized. Max {} steps for batch-size {}. \
        Shuffle {}".format(self.prediction_mode,datatype,max_steps,batch,shuffle))

    def eval(self,model,pairs,labels,epoch):
        def run_model(chem_repr, prot_repr, label, evaluate=True):
            chem_repr = load_from_mol_tuple(chem_repr)
            if isinstance(chem_repr, Variable) and torch.cuda.is_available():
                chem_repr = chem_repr.cuda()
            if isinstance(prot_repr, Variable) and torch.cuda.is_available():
                prot_repr = prot_repr.cuda()
            batch_input = {'protein': prot_repr, 'ligand': chem_repr}
            with torch.no_grad():
                logits = model(batch_input)
            batch_labels = torch.tensor(label)
            batch_labels = batch_labels.cpu().detach().numpy()
            logits = logits.cpu().detach().numpy()
            logging.debug("Evaluator-run_model: batch_labels {}, logits {}".format(batch_labels.shape, logits.shape))
            if evaluate:
                f1, auc, aupr = evaluate_binary_predictions(batch_labels, logits)
            else:
                return batch_labels, logits

        def get_repr_from_pairs_3(pairs):
            chem_repr = [(self.ikey2smiles[pair[0]], self.ikey2mol[pair[0]]) for pair in pairs]
            prot_repr = torch.stack(
                [torch.tensor(self.berttokenizer.encode(self.uniprot2triplets[pair[1]])) for pair in pairs])
            return (chem_repr, prot_repr)

        model.eval()

        datatype=self.datatype

        since=time.time()
        collected_logits = []
        collected_labels = []

        if len(pairs)<= self.batch:
            #sample size smaller than a batch
            chem_repr, prot_repr = get_repr_from_pairs(pairs)
            metrics = run_model(chem_repr,prot_repr,labels,evaluate=True)
            eval_time=time.time() - since
            logging.info("{:.2f} seconds for {} evaluation. Epoch {}".format(eval_time,datatype,epoch))
            print("{}\t{}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}".format(epoch,
                datatype,metrics[0],metrics[1],metrics[2]))
            return metrics

        elif len(pairs)<= self.max_steps*self.batch:
            #evaluate all pairs if small enough
            maxsteps=int(np.ceil(float(len(pairs))/float(self.batch)))
            for step in range(maxsteps):
                pairs_part=pairs[self.batch*step:self.batch*(step+1)]
                if len(pairs_part)==self.batch:
                    labels_part=labels[self.batch*step:self.batch*(step+1)]
                    chem_repr, prot_repr = get_repr_from_pairs_3(pairs_part)
                    # chem_repr,prot_repr = get_repr_from_pairs(pairs_part)
                    batch_labels,logits=run_model(chem_repr,prot_repr,labels_part,evaluate=False)
                    logging.debug("in-loop: batch_labels {}, logits {}".format(batch_labels.shape,logits.shape))
                    collected_logits.append(logits)
                    collected_labels.append(batch_labels)

        else:
            idxs = np.arange(len(pairs))
            np.random.shuffle(idxs)
            for step in range(self.max_steps):
                pairs_part=pairs[self.batch*step:self.batch*(step+1)]
                labels_part=labels[self.batch*step:self.batch*(step+1)]
                chem_repr, prot_repr = get_repr_from_pairs(pairs_part)
                batch_labels,logits=run_model(chem_repr,prot_repr,labels_part,evaluate=False)
                logging.debug("in-loop: batch_labels {}, logits {}".format(batch_labels.shape,logits.shape))
                collected_logits.append(logits)
                collected_labels.append(batch_labels)

        collected_labels = np.concatenate(collected_labels,axis=0)
        collected_logits = np.concatenate(collected_logits,axis=0)
        if self.prediction_mode.lower() in ['binary']:
            metrics = evaluate_binary_predictions(collected_labels,collected_logits)
        else:
            metrics = evaluate_continuous_predictions(collected_labels,collected_logits)

        eval_time=time.time() - since
        logging.info("{:.2f} seconds for {} evaluation. Epoch {}".format(eval_time,datatype,epoch))
        print("{}\t{}\t{:.5f}\t{:.5f}\t{:.5f}".format(epoch,datatype,metrics[0],metrics[1],metrics[2]))
        return metrics

def evaluate_binary_predictions(label,predprobs):
    logging.debug("label {}, predprobs {}".format(label.shape,predprobs.shape))
    probs=np.array(predprobs)
    predclass=np.argmax(probs,axis=1)

    f1 = metrics.f1_score(label, predclass, average='weighted')
    fpr,  tpr, thresholds = metrics.roc_curve(label, probs[:,1], pos_label=1)
    auc=metrics.auc(fpr, tpr)
    prec, reca, thresholds = metrics.precision_recall_curve(label, probs[:,1], pos_label=1)
    aupr=metrics.auc(reca,prec)
    return f1,auc,aupr