text:
include("singletimestepsolution.jl");

function synthesize_program(model_name::String;
                            singlecell = false
                            )
  observations, user_events, grid_size = generate_observations(model_name)
  matrix, unformatted_matrix, object_decomposition, prev_used_rules = singletimestepsolution_matrix(observations, user_events, grid_size, singlecell=singlecell)
  solutions = generate_on_clauses(matrix, unformatted_matrix, object_decomposition, user_events, grid_size)
  program_strings = []
  for solution in solutions
    if solution[1] != []
      on_clauses, new_object_decomposition, global_var_dict = solution
      program = full_program_given_on_clauses(on_clauses, new_object_decomposition, global_var_dict, grid_size)
      push!(program_strings, program)
    end
  end
  program_strings
end

function synthesis_program_pedro(observations::AbstractArray)

end

function generate_observations(model_name::String)
  program_expr = compiletojulia(parseautumn(programs[model_name]))
  m = eval(program_expr)
  if model_name == "particles"
    observations, user_events, grid_size = generate_observations_particles(m)
  elseif model_name == "ants"
    observations, user_events, grid_size = generate_observations_ants(m)
  elseif model_name == "lights"
    observations, user_events, grid_size = generate_observations_lights(m)
  elseif model_name == "water_plug"
    observations, user_events, grid_size = generate_observations_water_plug(m)
  elseif model_name == "ice"
    observations, user_events, grid_size = generate_observations_ice(m)
  elseif model_name == "tetris"
    observations, user_events, grid_size = generate_observations_tetris(m)
  elseif model_name == "snake"
    observations, user_events, grid_size = generate_observations_snake(m)
  elseif model_name == "magnets_i"
    observations, user_events, grid_size = generate_observations_magnets_i(m)
  elseif model_name == "magnets_ii"
    observations, user_events, grid_size = generate_observations_magnets_ii(m)
  elseif model_name == "magnets_iii"
    observations, user_events, grid_size = generate_observations_magnets_iii(m)
  elseif model_name == "disease"
    observations, user_events, grid_size = generate_observations_disease(m)
  elseif model_name == "space_invaders"
    observations, user_events, grid_size = generate_observations_space_invaders(m)
  elseif model_name == "gol"
    observations, user_events, grid_size = generate_observations_gol(m)
  elseif model_name == "sokoban_i"
    observations, user_events, grid_size = generate_observations_sokoban(m)
  elseif model_name == "sokoban_ii"
    observations, user_events, grid_size = generate_observations_sokoban_ii(m)
  elseif model_name == "grow"
    observations, user_events, grid_size = generate_observations_grow(m)
  elseif model_name == "mario"
    observations, user_events, grid_size = generate_observations_mario(m)
  elseif model_name == "sand"
    observations, user_events, grid_size = generate_observations_sand_simple(m)
  elseif model_name == "gravity_i"
    observations, user_events, grid_size = generate_observations_gravity(m)
  elseif model_name == "gravity_ii"
    observations, user_events, grid_size = generate_observations_gravity2(m)
  elseif model_name == "gravity_iii"
    observations, user_events, grid_size = generate_observations_gravity3(m)
  elseif model_name == "egg"
    observations, user_events, grid_size = generate_observations_egg(m)
  elseif model_name == "balloon"
    observations, user_events, grid_size = generate_observations_balloon(m)
  elseif model_name == "wind"
    observations, user_events, grid_size = generate_observations_wind(m)
  elseif model_name == "paint"
    observations, user_events, grid_size = generate_observations_paint(m)
  elseif model_name == "chase"
    observations, user_events, grid_size = generate_observations_chase(m)
  else
    error("model $(model_name) does not exist")
  end
end

programs = Dict("particles"                                 => """(program
                                                                  (= GRID_SIZE 16)

                                                                  (object Particle (Cell 0 0 "blue"))

                                                                  (: particles (List Particle))
                                                                  (= particles
                                                                    (initnext (list)
                                                                              (updateObj (prev particles) (--> obj (uniformChoice (list (moveLeft obj) (moveRight obj) (moveDown obj) (moveUp obj)) )))))

                                                                  (on clicked (= particles (addObj (prev particles) (Particle (Position (.. click x) (.. click y))))))
                                                                  )""",
                "chase"                                      => """(program
                                                                    (= GRID_SIZE 16)

                                                                    (object Ant (Cell 0 0 "red"))
                                                                    (object Agent (Cell 0 0 "green"))

                                                                    (: ants (List Ant))
                                                                    (= ants (initnext (list (Ant (Position 6 9)) )
                                                                                (prev ants)))

                                                                    (: agent Agent)
                                                                    (= agent (initnext (Agent (Position 7 5)) (prev agent)))

                                                                    (: time Int)
                                                                    (= time (initnext 0 (+ (prev time) 1)))

                                                                    (on true (= ants (updateObj (prev ants) (--> obj (move obj (unitVector obj (closest obj Agent)))) )))
                                                                    (on (== (% time 4) 2) (= ants (addObj ants (map Ant (randomPositions GRID_SIZE 1)))))

                                                                    (on left (= agent (moveLeft (prev agent))))
                                                                    (on right (= agent (moveRight (prev agent))))
                                                                    (on up (= agent (moveUp (prev agent))))
                                                                    (on down (= agent (moveDown (prev agent))))

                                                                    (on (intersects (prev agent) (prev ants)) (= agent (removeObj (prev agent))))
                                                                  )""",
                "lights"                                    => """(program
                                                                  (= GRID_SIZE 10)

                                                                  (object Light (: on Bool) (Cell 0 0 (if on then "yellow" else "white")))

                                                                  (: lights (List Light))
                                                                  (= lights (initnext (map (--> pos (Light false pos)) (filter (--> pos (== (.. pos x) (.. pos y))) (allPositions GRID_SIZE)))
                                                                                (prev lights)))

                                                                  (on clicked (= lights (updateObj lights (--> obj (updateObj obj "on" (! (.. obj on)))))))
                                                                )""",
                "water_plug"                                => """(program
                                                                  (= GRID_SIZE 16)

                                                                  (object Button (: color String) (Cell 0 0 color))
                                                                  (object Vessel (Cell 0 0 "purple"))
                                                                  (object Plug (Cell 0 0 "orange"))
                                                                  (object Water (Cell 0 0 "blue"))

                                                                  (: vesselButton Button)
                                                                  (= vesselButton (Button "purple" (Position 2 0)))
                                                                  (: plugButton Button)
                                                                  (= plugButton (Button "orange" (Position 5 0)))
                                                                  (: waterButton Button)
                                                                  (= waterButton (Button "blue" (Position 8 0)))
                                                                  (: removeButton Button)
                                                                  (= removeButton (Button "black" (Position 11 0)))
                                                                  (: clearButton Button)
                                                                  (= clearButton (Button "red" (Position 14 0)))

                                                                  (: vessels (List Vessel))
                                                                  (= vessels (initnext (list (Vessel (Position 6 15)) (Vessel (Position 6 14)) (Vessel (Position 6 13)) (Vessel (Position 5 12)) (Vessel (Position 4 11)) (Vessel (Position 3 10)) (Vessel (Position 9 15)) (Vessel (Position 9 14)) (Vessel (Position 9 13)) (Vessel (Position 10 12)) (Vessel (Position 11 11)) (Vessel (Position 12 10))) (prev vessels)))
                                                                  (: plugs (List Plug))
                                                                  (= plugs (initnext (list (Plug (Position 7 15)) (Plug (Position 8 15)) (Plug (Position 7 14)) (Plug (Position 8 14)) (Plug (Position 7 13)) (Plug (Position 8 13))) (prev plugs)))
                                                                  (: water (List Water))
                                                                  (= water (initnext (list) (updateObj (prev water) nextLiquid)))

                                                                  (= currentParticle (initnext "vessel" (prev currentParticle)))

                                                                  (on (& clicked (& (isFree click) (== currentParticle "vessel"))) (= vessels (addObj (prev vessels) (Vessel (Position (.. click x) (.. click y))))))
                                                                  (on (& clicked (& (isFree click) (== currentParticle "plug"))) (= plugs (addObj (prev plugs) (Plug (Position (.. click x) (.. click y))))))
                                                                  (on (& clicked (& (isFree click) (== currentParticle "water"))) (= water (addObj (prev water) (Water (Position (.. click x) (.. click y))))))
                                                                  (on (clicked vesselButton) (= currentParticle "vessel"))
                                                                  (on (clicked plugButton) (= currentParticle "plug"))
                                                                  (on (clicked waterButton) (= currentParticle "water"))
                                                                  (on (clicked removeButton) (= plugs (removeObj plugs (--> obj true))))
                                                                  (on (clicked clearButton) (let ((= vessels (removeObj vessels (--> obj true))) (= plugs (removeObj plugs (--> obj true))) (= water (removeObj water (--> obj true))))))
                                                                )""",
                "ice"                                       => """(program
                                                                  (= GRID_SIZE 8)
                                                                  (object CelestialBody (: day Bool) (list (Cell 0 0 (if day then "gold" else "gray"))
                                                                                (Cell 0 1 (if day then "gold" else "gray"))
                                                                                (Cell 1 0 (if day then "gold" else "gray"))
                                                                                (Cell 1 1 (if day then "gold" else "gray"))))
                                                                  (object Cloud (list (Cell -1 0 "gray")
                                                                                (Cell 0 0 "gray")
                                                                                (Cell 1 0 "gray")))

                                                                  (object Water (: liquid Bool) (Cell 0 0 (if liquid then "blue" else "lightblue")))

                                                                  (: celestialBody CelestialBody)
                                                                  (= celestialBody (initnext (CelestialBody true (Position 0 0)) (prev celestialBody)))

                                                                  (: cloud Cloud)
                                                                  (= cloud (initnext (Cloud (Position 4 0)) (prev cloud)))

                                                                  (: water (List Water))
                                                                  (= water (initnext (list) (updateObj (prev water) nextWater)))

                                                                  (on left (= cloud (nextCloud cloud (Position -1 0))))
                                                                  (on right (= cloud (nextCloud cloud (Position 1 0))))
                                                                  (on down (= water (addObj water (Water (.. celestialBody day) (move (.. cloud origin) (Position 0 1))))))
                                                                  (on clicked (let ((= celestialBody (updateObj celestialBody "day" (! (.. celestialBody day)))) (= water (updateObj water (--> drop (updateObj drop "liquid" (! (.. drop liquid)))))))))

                                                                  (= nextWater (fn (drop)
                                                                                (if (.. drop liquid)
                                                                                then (nextLiquid drop)
                                                                                else (nextSolid drop))))

                                                                  (= nextCloud (fn (cloud position)
                                                                                (if (isWithinBounds (move cloud position))
                                                                                then (move cloud position)
                                                                                else cloud)))
                                                                )""",
                "tetris" => ""
                ,"snake" => ""
                ,"magnets_i" => ""
                ,"magnets_ii" => ""
                ,"magnets_iii" => ""
                ,"disease"                                    => """(program
                                                                (= GRID_SIZE 8)

                                                                (object Particle (: health Bool) (Cell 0 0 (if health then "gray" else "darkgreen")))

                                                                (: inactiveParticles (List Particle))
                                                                (= inactiveParticles (initnext (list (Particle true (Position 5 3)) (Particle true (Position 2 1)) (Particle true (Position 4 4)) (Particle true (Position 1 3)) )
                                                                                (updateObj (prev inactiveParticles) (--> obj (if (! (.. activeParticle health))
                                                                                then (updateObj obj "health" false)
                                                                                else obj))
                                                                                (--> obj (adjacent (.. obj origin) (.. (prev activeParticle) origin))))))

                                                                (: activeParticle Particle)
                                                                (= activeParticle (initnext (Particle false (Position 0 0)) (prev activeParticle)))

                                                                (on (!= (length (filter (--> obj (! (.. obj health))) (adjacentObjs activeParticle))) 0) (= activeParticle (updateObj (prev activeParticle) "health" false)))
                                                                (on (clicked (prev inactiveParticles))
                                                                    (let ((= inactiveParticles (addObj (prev inactiveParticles) (prev activeParticle)))
                                                                          (= activeParticle (objClicked click (prev inactiveParticles)))
                                                                          (= inactiveParticles (removeObj inactiveParticles (objClicked click (prev inactiveParticles))))
                                                                        )))
                                                                (on left (= activeParticle (moveNoCollision (prev activeParticle) -1 0)))
                                                                (on right (= activeParticle (moveNoCollision (prev activeParticle) 1 0)))
                                                                (on up (= activeParticle (moveNoCollision (prev activeParticle) 0 -1)))
                                                                (on down (= activeParticle (moveNoCollision (prev activeParticle) 0 1)))
                                                              )"""
                ,"space_invaders" => ""
                ,"gol" => ""
                ,"sokoban_i" =>                               """(program
                                                                  (= GRID_SIZE 8)

                                                                  (object Agent (Cell 0 0 "blue"))
                                                                  (object Box (Cell 0 0 "black"))
                                                                  (object Goal (Cell 0 0 "red"))

                                                                  (: agent Agent)
                                                                  (= agent (initnext (Agent (Position 7 4)) (prev agent)))

                                                                  (: boxes (List Box))
                                                                  (= boxes (initnext (list (Box (Position 1 2)) (Box (Position 0 4)) (Box (Position 4 4))) (prev boxes)))

                                                                  (: goal Goal)
                                                                  (= goal (initnext (Goal (Position 0 0)) (prev goal)))

                                                                  (on left (let ((= boxes (moveBoxes (prev boxes) (prev agent) (prev goal) -1 0))
                                                                                (= agent (moveAgent (prev agent) (prev boxes) (prev goal) -1 0)))))

                                                                  (on right (let ((= boxes (moveBoxes (prev boxes) (prev agent) (prev goal) 1 0))
                                                                                (= agent (moveAgent (prev agent) (prev boxes) (prev goal) 1 0)))))

                                                                  (on up (let ((= boxes (moveBoxes (prev boxes) (prev agent) (prev goal) 0 -1))
                                                                              (= agent (moveAgent (prev agent) (prev boxes) (prev goal) 0 -1)))))

                                                                  (on down (let ((= boxes (moveBoxes (prev boxes) (prev agent) (prev goal) 0 1))
                                                                                (= agent (moveAgent (prev agent) (prev boxes) (prev goal) 0 1)))))

                                                                  (on (& clicked (isFree click)) (= boxes (addObj boxes (Box (Position (.. click x) (.. click y))))))

                                                                  (: moveBoxes (-> (List Box) Agent Goal Int Int (List Box)))
                                                                  (= moveBoxes (fn (boxes agent goal x y)
                                                                                (updateObj boxes
                                                                                (--> obj (if (intersects (move obj x y) goal) then (removeObj obj) else (moveNoCollision obj x y)))
                                                                                (--> obj (== (displacement (.. obj origin) (.. agent origin)) (Position (- 0 x) (- 0 y)))))))

                                                                  (: moveAgent (-> Agent (List Box) Goal Int Int Agent))
                                                                  (= moveAgent (fn (agent boxes goal x y)
                                                                                (if (| (intersects (list (move agent x y)) (moveBoxes boxes agent goal x y))
                                                                                (! (isWithinBounds (move agent x y))))
                                                                                then agent
                                                                                else (move agent x y))))
                                                                )"""
                ,"sokoban_ii" => ""
                ,"grow"                                      => """(program
                                                                  (= GRID_SIZE 8)

                                                                  (object Water (Cell 0 0 "blue"))
                                                                  (object Leaf (: color String) (Cell 0 0 color))
                                                                  (object Cloud (list (Cell -1 0 "gray") (Cell 0 0 "gray") (Cell 1 0 "gray")
                                                                                (Cell -1 1 "gray") (Cell 0 1 "gray") (Cell 1 1 "gray")))

                                                                  (object Sun (: movingLeft Bool) (list (Cell 0 0 "gold")
                                                                                (Cell 0 1 "gold")
                                                                                (Cell 1 0 "gold")
                                                                                (Cell 1 1 "gold")))

                                                                  (: sun Sun)
                                                                  (= sun (initnext (Sun false (Position 0 0)) (prev sun)))

                                                                  (: water (List Water))
                                                                  (= water (initnext (list) (updateObj (prev water) (--> obj (if (! (isWithinBounds obj)) then (removeObj obj) else (moveDown obj))))))

                                                                  (: cloud Cloud)
                                                                  (= cloud (initnext (Cloud (Position 6 0)) (prev cloud)))

                                                                  (: leaves (List Leaf))
                                                                  (= leaves (initnext (list (Leaf "green" (Position 1 7) ) (Leaf "green" (Position 3 7)) (Leaf "green" (Position 5 7)) ) (prev leaves)))

                                                                  (on (intersects (map (--> obj (moveDown obj)) (prev water) ) (prev leaves))
                                                                    (= water (removeObj (prev water) (--> obj (intersects (moveDown obj) (prev leaves))) ) ) )

                                                                  (on (& (intersects (map (--> obj (moveDown obj)) (prev water)) (filter (--> obj (== (.. obj color) "green")) (prev leaves))) (! (intersects (prev sun) (prev cloud))))
                                                                    (= leaves (addObj (prev leaves) (map (--> obj (Leaf (if (== (.. (.. (moveUp obj) origin) y) 4) then "mediumpurple" else "green") (.. (moveUp obj) origin))) (filter (--> obj (intersects (moveUp obj) (prev water))) (prev leaves))))))

                                                                  (on down
                                                                    (= water (addObj water (Water (.. (moveDown (prev cloud)) origin)))))

                                                                  (on left (= cloud (moveLeft (prev cloud))))
                                                                  (on right (= cloud (moveRight (prev cloud))))

                                                                  (on (== (.. (.. (prev sun) origin) x) 0) (= sun (updateObj (prev sun) "movingLeft" false)))
                                                                  (on (== (.. (.. (prev sun) origin) x) 6) (= sun (updateObj (prev sun) "movingLeft" true)))

                                                                  (on (clicked (prev sun)) (= sun (if (.. (prev sun) movingLeft) then (moveLeft (prev sun)) else (moveRight (prev sun)))))
                                                                )"""
                ,"mario" => ""
                ,"sand"                                      => """(program
                                                                  (= GRID_SIZE 10)

                                                                  (object Button (: color String) (Cell 0 0 color))
                                                                  (object Sand (: liquid Bool) (Cell 0 0 (if liquid then "sandybrown" else "tan")))
                                                                  (object Water (Cell 0 0 "skyblue"))

                                                                  (: sandButton Button)
                                                                  (= sandButton (initnext (Button "tan" (Position 2 0)) (prev sandButton)))

                                                                  (: waterButton Button)
                                                                  (= waterButton (initnext (Button "skyblue" (Position 7 0)) (prev waterButton)))

                                                                  (: sand (List Sand))
                                                                  (= sand (initnext (list)
                                                                            (updateObj (prev sand) (--> obj (if (.. obj liquid)
                                                                                then (nextLiquid obj)
                                                                                else (nextSolid obj))))))

                                                                  (: water (List Water))
                                                                  (= water (initnext (list) (updateObj (prev water) (--> obj (nextLiquid obj)))))

                                                                  (: clickType String)
                                                                  (= clickType (initnext "sand" (prev clickType)))

                                                                  (on true (= sand (updateObj (prev sand) (--> obj (updateObj obj "liquid" true)) (--> obj (& (! (.. obj liquid)) (intersects (adjacentObjs obj) (prev water)))))))

                                                                  (on (clicked sandButton) (= clickType "sand"))
                                                                  (on (clicked waterButton) (= clickType "water"))
                                                                  (on (& (& clicked (isFree click)) (== clickType "sand"))  (= sand (addObj sand (Sand false (Position (.. click x) (.. click y))))))
                                                                  (on (& (& clicked (isFree click)) (== clickType "water")) (= water (addObj water (Water (Position (.. click x) (.. click y))))))

                                                                )"""
                ,"gravity_i"                                 => """(program
                                                                  (= GRID_SIZE 16)

                                                                  (object Button (: color String) (Cell 0 0 color))
                                                                  (object Blob (list (Cell 0 -1 "blue") (Cell 0 0 "blue") (Cell 1 -1 "blue") (Cell 1 0 "blue")))

                                                                  (: leftButton Button)
                                                                  (= leftButton (initnext (Button "red" (Position 0 7)) (prev leftButton)))

                                                                  (: rightButton Button)
                                                                  (= rightButton (initnext (Button "darkorange" (Position 15 7)) (prev rightButton)))

                                                                  (: upButton Button)
                                                                  (= upButton (initnext (Button "gold" (Position 7 0)) (prev upButton)))

                                                                  (: downButton Button)
                                                                  (= downButton (initnext (Button "green" (Position 7 15)) (prev downButton)))

                                                                  (: blobs (List Blob))
                                                                  (= blobs (initnext (list) (prev blobs)))

                                                                  (: gravity String)
                                                                  (= gravity (initnext "down" (prev gravity)))

                                                                  (on (== gravity "left") (= blobs (updateObj blobs (--> obj (moveLeftNoCollision obj)))))
                                                                  (on (== gravity "right") (= blobs (updateObj blobs (--> obj (moveRightNoCollision obj)))))
                                                                  (on (== gravity "up") (= blobs (updateObj blobs (--> obj (moveUpNoCollision obj)))))
                                                                  (on (== gravity "down") (= blobs (updateObj blobs (--> obj (moveDownNoCollision obj)))))

                                                                  (on (& clicked (isFree click)) (= blobs (addObj blobs (Blob (Position (.. click x) (.. click y))))) )

                                                                  (on (clicked leftButton) (= gravity "left"))

                                                                  (on (clicked rightButton) (= gravity "right"))

                                                                  (on (clicked upButton) (= gravity "up"))

                                                                  (on (clicked downButton) (= gravity "down"))
                                                                )"""
                ,"gravity_ii"                                => """(program
                                                                    (= GRID_SIZE 30)

                                                                    (object Button (: color String) (Cell 0 0 color))
                                                                    (object Blob (: color String) (list (Cell 0 -1 color) (Cell 0 0 color) (Cell 1 -1 color) (Cell 1 0 color)))

                                                                    (: leftButton Button)
                                                                    (= leftButton (initnext (Button "red" (Position 0 14)) (prev leftButton)))

                                                                    (: rightButton Button)
                                                                    (= rightButton (initnext (Button "darkorange" (Position 29 14)) (prev rightButton)))

                                                                    (: upButton Button)
                                                                    (= upButton (initnext (Button "gold" (Position 14 0)) (prev upButton)))

                                                                    (: downButton Button)
                                                                    (= downButton (initnext (Button "green" (Position 14 29)) (prev downButton)))

                                                                    (: blobs (List Blob))
                                                                    (= blobs (initnext (list) (prev blobs)))

                                                                    (: gravity String)
                                                                    (= gravity (initnext "down" (prev gravity)))

                                                                    (: blobColor Int)
                                                                    (= blobColor (initnext 0 (prev blobColor)))

                                                                    (on (== gravity "left") (= blobs (updateObj (prev blobs) (--> obj (moveLeftNoCollision obj)))))
                                                                    (on (== gravity "right") (= blobs (updateObj (prev blobs) (--> obj (moveRightNoCollision obj)))))
                                                                    (on (== gravity "up") (= blobs (updateObj (prev blobs) (--> obj (moveUpNoCollision obj)))))
                                                                    (on (== gravity "down") (= blobs (updateObj (prev blobs) (--> obj (moveDownNoCollision obj)))))

                                                                    (on (& clicked (isFree click)) (= blobColor (% (+ (prev blobColor) 1) 3)))
                                                                    (on (& (& clicked (isFree click)) (== blobColor 0)) (= blobs (addObj blobs (Blob "blue" (Position (.. click x) (.. click y))))))
                                                                    (on (& (& clicked (isFree click)) (== blobColor 1)) (= blobs (addObj blobs (Blob "mediumpurple" (Position (.. click x) (.. click y))))))
                                                                    (on (& (& clicked (isFree click)) (== blobColor 2)) (= blobs (addObj blobs (Blob "magenta" (Position (.. click x) (.. click y))))))

                                                                    (on (clicked leftButton) (= gravity "left"))

                                                                    (on (clicked rightButton) (= gravity "right"))

                                                                    (on (clicked upButton) (= gravity "up"))

                                                                    (on (clicked downButton) (= gravity "down"))
                                                                  )"""
                ,"gravity_iii"                                 => """(program
                                                                    (= GRID_SIZE 19)
                                                                    (= background "black")

                                                                    (object Button (: color String) (Cell 0 0 color))
                                                                    (object Blob (list (Cell 0 0 "blue")))

                                                                    (: blobs (List Blob))
                                                                    (= blobs (initnext (list (Blob (Position 9 9))) (prev blobs)))

                                                                    (: xVel Int)
                                                                    (= xVel (initnext 0 (prev xVel)))

                                                                    (: yVel Int)
                                                                    (= yVel (initnext 0 (prev yVel)))

                                                                    (on (& clicked (isFree click)) (= blobs (addObj blobs (Blob (Position (.. click x) (.. click y))))))

                                                                    (on (& left (!= (prev xVel) -1)) (= xVel (- (prev xVel) 1)))
                                                                    (on (& right (!= (prev xVel) 1)) (= xVel (+ (prev xVel) 1)))

                                                                    (on (& up (!= (prev yVel) -1)) (= yVel (- (prev yVel) 1)))
                                                                    (on (& down (!= (prev yVel) 1)) (= yVel (+ (prev yVel) 1)))

                                                                    (on true (= blobs (updateObj blobs (--> obj (moveNoCollision obj (Position xVel yVel))))))
                                                                  )"""
                ,"egg"                                         => """(program
                                                                    (= GRID_SIZE 16)
                                                                    (= background "black")

                                                                    (object Button (: color String) (Cell 0 0 color))
                                                                    (object Piece (Cell 0 0 "gold"))
                                                                    (object Egg (list (Cell -1 -2 "tan") (Cell 0 -2 "tan") (Cell 1 -2 "tan")
                                                                                (Cell -2 -1 "tan") (Cell -1 -1 "tan") (Cell 0 -1 "tan") (Cell 1 -1 "tan") (Cell 2 -1 "tan")
                                                                                (Cell -2 0 "tan") (Cell -1 0 "tan") (Cell 0 0 "tan") (Cell 1 0 "tan") (Cell 2 0 "tan")
                                                                                (Cell -2 1 "tan") (Cell -1 1 "tan") (Cell 0 1 "tan") (Cell 1 1 "tan") (Cell 2 1 "tan")
                                                                                (Cell -1 2 "tan") (Cell 0 2 "tan") (Cell 1 2 "tan")))

                                                                    (: egg Egg)
                                                                    (= egg (initnext (Egg (Position 7 13)) (prev egg)))

                                                                    (: pieces (List Piece))
                                                                    (= pieces (initnext (list) (updateObj (prev pieces) (--> obj (nextLiquid obj)))))

                                                                    (: button Button)
                                                                    (= button (initnext (Button "red" (Position 0 0)) (updateObj (prev button) "color" (if gravity then "pink" else "red"))))

                                                                    (: gravity Bool)
                                                                    (= gravity (initnext false (prev gravity)))

                                                                    (: height Int)
                                                                    (= height (initnext 15 (prev height)))

                                                                    (on gravity (= egg (moveDownNoCollision (prev egg))))

                                                                    (on (& left (! gravity)) (= egg (moveLeftNoCollision (prev egg))))
                                                                    (on (& right (! gravity)) (= egg (moveRightNoCollision (prev egg))))
                                                                    (on (& up (! gravity)) (= egg (moveUpNoCollision (prev egg))))
                                                                    (on (& down (! gravity)) (= egg (moveDownNoCollision (prev egg))))

                                                                    (on (clicked (prev button))
                                                                      (let ((= gravity (! (prev gravity)))
                                                                            (= height (.. (.. (prev egg) origin) y)))))

                                                                    (on (& (< height 7) (== (.. (.. egg origin) y) 13))
                                                                      (let ((= egg (removeObj egg))
                                                                            (= pieces (addObj (prev pieces) (map (--> obj (Piece (Position (.. (.. obj position) x) (+ 1 (.. (.. obj position) y))))) (render (prev egg))))))))

                                                                  )"""
                ,"balloon" => ""
              ,"wind"                                          => """(program
                                                                      (= GRID_SIZE 17)
                                                                      (= background "darkblue")

                                                                      (object Water (Cell 0 0 "lightblue"))
                                                                      (object Cloud (map (--> pos (Cell (.. pos x) (.. pos y) "gray")) (rect (Position 0 0) (Position 16 1))) (prev cloud))

                                                                      (: water (List Water))
                                                                      (= water (initnext (list) (prev water)))

                                                                      (: cloud Cloud)
                                                                      (= cloud (initnext (Cloud (Position 0 0)) (prev cloud)))

                                                                      (: wind Int)
                                                                      (= wind (initnext 0 (prev wind)))

                                                                      (: time Int)
                                                                      (= time (initnext 0 (+ (prev time) 1)))

                                                                      (on (== wind 0) (= water (updateObj (prev water) (--> obj (moveDown obj)))))
                                                                      (on (== wind 1) (= water (updateObj (prev water) (--> obj (moveRight (moveDown obj))))))
                                                                      (on (== wind -1) (= water (updateObj (prev water) (--> obj (moveLeft (moveDown obj))))))

                                                                      (on left (= wind (if (== (prev wind) -1) then (prev wind) else (- (prev wind) 1))))
                                                                      (on right (= wind (if (== (prev wind) 1) then (prev wind) else (+ (prev wind) 1))))

                                                                      (on (== (% time 5) 2)
                                                                        (= water (addObj
                                                                                water
                                                                                (map (--> pos (Water pos)) (list (Position 2 2)

                                                                                (Position 6 2)

                                                                                (Position 10 2)

                                                                                (Position 14 2)))))))"""
                ,"paint"                                         => """(program
                                                                      (= GRID_SIZE 16)

                                                                      (object Particle (: color String) (Cell 0 0 color))

                                                                      (: particles (List Particle))
                                                                      (= particles (initnext (list) (prev particles)))

                                                                      (: currColor String)
                                                                      (= currColor (initnext "red" (prev currColor)))

                                                                      (on (& clicked (isFree click)) (= particles (addObj (prev particles) (Particle currColor (Position (.. click x) (.. click y))))))
                                                                      (on (& up (== (prev currColor) "red")) (= currColor "gold"))
                                                                      (on (& up (== (prev currColor) "gold")) (= currColor "green"))
                                                                      (on (& up (== (prev currColor) "green")) (= currColor "blue"))
                                                                      (on (& up (== (prev currColor) "blue")) (= currColor "purple"))
                                                                      (on (& up (== (prev currColor) "purple")) (= currColor "red"))
                                                                    )"""
                ,"ants" =>                                        """(program
                                                                      (= GRID_SIZE 16)

                                                                      (object Ant (Cell 0 0 "gray"))
                                                                      (object Food (Cell 0 0 "red"))

                                                                      (: ants (List Ant))
                                                                      (= ants (initnext (map Ant (randomPositions GRID_SIZE 1)) (prev ants)))

                                                                      (: foods (List Food))
                                                                      (= foods (initnext (list) (prev foods)))

                                                                      (on true (= ants (updateObj (prev ants) (--> obj (move obj (unitVector obj (closest obj Food)))))))
                                                                      (on true (= foods (updateObj (prev foods) (--> obj (if (intersects obj (prev ants))
                                                                                then (removeObj obj)
                                                                                else obj)))))

                                                                      (on clicked (= foods (addObj foods (map Food (randomPositions GRID_SIZE 4)))))
                                                                    )"""
                )


text:
using ClimateMachine
const clima_dir = dirname(dirname(pathof(ClimateMachine)));

if parse(Bool, get(ENV, "CLIMATEMACHINE_PLOT_EDMF_COMPARISON", "false"))
    plot_dir = joinpath(clima_dir, "output", "sbl_edmf", "pycles_comparison")
else
    plot_dir = nothing
end

include(joinpath(@__DIR__, "compute_mse.jl"))

data_file = Dataset(joinpath(PyCLES_output_dataset_path, "Gabls.nc"), "r")

#! format: off
best_mse = OrderedDict()
best_mse["prog_ρ"] = 7.9878085788692155e-03
best_mse["prog_ρu_1"] = 3.0923908085978383e+03
best_mse["prog_ρu_2"] = 4.4228530597867703e+01
#! format: on

computed_mse = compute_mse(
    solver_config.dg.grid,
    solver_config.dg.balance_law,
    time_data,
    dons_arr,
    data_file,
    "Gabls",
    best_mse,
    3600 * 6,
    plot_dir,
)

@testset "SBL Implicit Solution Quality Assurance (QA) tests" begin
    #! format: off
    test_mse(computed_mse, best_mse, "prog_ρ")
    test_mse(computed_mse, best_mse, "prog_ρu_1")
    test_mse(computed_mse, best_mse, "prog_ρu_2")
    #! format: on
end

ext: jl


text:
type Grid{T,N,d,P<:AbstractArray,Q<:AbstractArray} <: AbstractArray{T,N}
    c::P
    w::Q
    function Grid{V}(c::AbstractArray{T,N},w::AbstractArray{V,N})
        @assert size(c) == size(w)
        new(c,w)
    end
end

Grid{T<:Tuple}(c::AbstractArray{T},w::AbstractArray) = Grid{eltype(c),ndims(c),length(T.types),typeof(c),typeof(w)}(c,w)
Grid{T<:Number}(c::AbstractArray{T},w::AbstractArray) = Grid{eltype(c),ndims(c),1,typeof(c),typeof(w)}(c,w)

Base.length(g::Grid) = length(g.c)
Base.eltype{T,N}(g::Grid{T,N}) = T
Base.ndims{T,N}(g::Grid{T,N}) = N
Base.size(g::Grid) = size(g.c)
Base.size(g::Grid, i::Integer) = size(g.c, i)
Base.getindex(g::Grid, i...) = g.c[i...]
function Base.setindex!(g::Grid, value, i...)
    g.c[i...] = value
end
Base.show(io::IO, g::Grid) = print(io, summary(g))
Base.writemime(io::IO, mime::MIME"text/plain", g::Grid) = writemime(io, mime, g.c)
Base.transpose(g::Grid) = transpose(g.c)
Base.ctranspose(g::Grid) = ctranspose(g.c)
Base.convert(T::Type{Array{Float64,1}},g::Grid) = convert(T,g.c)
Base.eachindex(g::Grid) = eachindex(g.c)

integrate(g::Grid, f::AbstractVector) = vecdot(g.w, f)
integrate(g::Grid, f::AbstractArray) = sum(g.w .* f, 1:ndims(g.w)) |> squeezeall
integrate(g::Grid, f) = integrate(g, f(g))

function integrate(g::Grid, f::AbstractArray, dims::Integer)
    @assert length(dims) == ndims(g.w)
    shp = fill(1,ndims(f))
    for (i,d) in enumerate(dims)
        shp[d] = size(g.w,i)
    end
    sum(reshape(g.w,tuple(shp...)) .* f, dims) |> squeezeall
end

Base.(:*){T}(A::Array{T,2}, g::Grid{Point3{T}}) = Grid(reshape(Point3{T}[A*x for x in g.c], size(g.c)), g.w)

# function expand(grid, basis, f)
#     S = basis.data' * (basis .* x.w)
#     (integrate(x, basis.*h)' / S)'
# end

# function expand(grid, basis, f)
#     S = Symmetric(LinAlg.BLAS.syrk('U', 'T', basis.data .* sqrt(grid.w)))
#     (integrate(grid, basis.*f)' / S)'
# end

# LinAlg.BLAS.herk('U', 'C', 1.0, mat(basis.data .* sqrt(g.w)))

# function qrgramschmidt(grid, basis)
#     Q,R = qr(basis .* sqrt(grid.w));
#     Q ./= sqrt(grid.w)
#     Q
# end

function lingrid(start::Real, stop::Real, n::Integer)
    x = collect(linspace(start, stop, n))
    dx = x[2] - x[1]
    Grid(x, fill(dx, length(x)))
end

function loggrid(start::Real, stop::Real, n::Integer)
    x = collect(linspace(start, stop, n))
    dx = x[2] - x[1]
    r = exp(x)
    Grid(r, r.*dx)
end

function simpsgrid(start::Real, stop::Real, n::Integer)
    @assert isodd(n) "Simpsons Rule requires an odd number of grid points"
    x = collect(linspace(start, stop, n))
    dx = x[2] - x[1]
    w = fill(dx/3, n)
    w[2:2:end] .*= 4
    w[3:2:end-1] .*= 2
    Grid(x, w)
end

function atomicradgrid(Nr::Int, rmax::Real)
    s = collect(1:Nr)
    r = rmax * log(1 - (s/(Nr+1.)).^2) ./ log(1 - (Nr/(Nr+1))^2)
    w = -2rmax*s ./ ( (1+Nr-s) .* (1+Nr+s) .* log(1 - (Nr/(Nr+1.))^2) )
    Grid(r, w)
end

lebgrid(n::Int) = lebedev[n]

function atomicgrid(r::Grid, Ω::Grid; R::Point3{Float64}=(0.,0.,0.))
    shp = (length(r),length(Ω))
    g = Grid(Array{Point3{Float64}}(shp), Array{Float64}(shp))
    for i in 1:length(Ω)
        θ,ϕ = Ω[i]
        for j in 1:length(r)
            g.w[j,i] = r[j]^2*r.w[j]*Ω.w[i]
            g.c[j,i] = sph2cart(r[j],θ,ϕ) + R
        end
    end
    g
end

# function multigrid(atoms)
#     for at in atoms
#         at.p = at.h(at.g)./sum([μ.h(at.g) for μ in atoms])
#     end
#     p = cat([ν.p for ν in atoms]...)
#     g = Grid(cat([μ.g for μ in atoms]...), p.*cat([μ.g.w for μ in atoms]...))
#     for at in atoms
#         at.rho = at.rho(g)
#         at.vh = at.vh(g)
#         at.v = at.v(g)
#     end
#     g
# end

max_stars_repo_name: johncsnyder/Element


text:
# This file is used for code coverage of Causal test suites.

# NOTE: Before executing this script, run test suite of Causal as
# ] test --coverage Causal
#

using Coverage

# process '*.cov' files
coverage = process_folder() # defaults to src/; alternatively, supply the folder name as argument
coverage = append!(coverage, process_folder("deps"))

# process '*.info' files
coverage = merge_coverage_counts(coverage, filter!(
    let prefixes = (joinpath(pwd(), "src", ""),
                    joinpath(pwd(), "deps", ""))
        c -> any(p -> startswith(c.filename, p), prefixes)
    end,
    LCOV.readfolder("test")))

# Get total coverage for all Julia files
covered_lines, total_lines = get_summary(coverage)
percentage = covered_lines / total_lines * 100
@info "Code coverage percentage : $percentage%"

# Clean folders
foreach(clean_folder, ["src", "test"])


text:
betweenness_centrality(g::AbstractGraph, vs::AbstractVector=vertices(g), distmx::AbstractMatrix=weights(g); normalize=true, endpoints=false, parallel=:distributed) =
parallel == :distributed ? distr_betweenness_centrality(g, vs, distmx; normalize=normalize, endpoints=endpoints) :
threaded_betweenness_centrality(g, vs, distmx; normalize=normalize, endpoints=endpoints)

betweenness_centrality(g::AbstractGraph, k::Integer, distmx::AbstractMatrix=weights(g); normalize=true, endpoints=false, parallel=:distributed) =
parallel == :distributed ? distr_betweenness_centrality(g, sample(vertices(g), k), distmx; normalize=normalize, endpoints=endpoints) :
threaded_betweenness_centrality(g, sample(vertices(g), k), distmx; normalize=normalize, endpoints=endpoints)

function distr_betweenness_centrality(g::AbstractGraph,
    vs::AbstractVector=vertices(g),
    distmx::AbstractMatrix=weights(g);
    normalize=true,
    endpoints=false)::Vector{Float64}

    n_v = nv(g)
    k = length(vs)
    isdir = is_directed(g)

    # Parallel reduction

    betweenness = @distributed (+) for s in vs
        temp_betweenness = zeros(n_v)
        if degree(g, s) > 0  # this might be 1?
            state = LightGraphs.dijkstra_shortest_paths(g, s, distmx; allpaths=true, trackvertices=true)
            if endpoints
                LightGraphs._accumulate_endpoints!(temp_betweenness, state, g, s)
            else
                LightGraphs._accumulate_basic!(temp_betweenness, state, g, s)
            end
        end
        temp_betweenness
    end

    LightGraphs._rescale!(betweenness,
    n_v,
    normalize,
    isdir,
    k)

    return betweenness
end

distr_betweenness_centrality(g::AbstractGraph, k::Integer, distmx::AbstractMatrix=weights(g); normalize=true, endpoints=false) =
    distr_betweenness_centrality(g, sample(vertices(g), k), distmx; normalize=normalize, endpoints=endpoints)

function threaded_betweenness_centrality(g::AbstractGraph,
    vs::AbstractVector=vertices(g),
    distmx::AbstractMatrix=weights(g);
    normalize=true,
    endpoints=false)::Vector{Float64}

    n_v = nv(g)
    k = length(vs)
    isdir = is_directed(g)

    local_betweenness = [zeros(n_v) for i in 1:nthreads()]
    vs_active = findall((x)->degree(g, x) > 0, vs) # 0 might be 1?

    Base.Threads.@threads for s in vs_active
        state = LightGraphs.dijkstra_shortest_paths(g, s, distmx; allpaths=true, trackvertices=true)
        if endpoints
            LightGraphs._accumulate_endpoints!(local_betweenness[Base.Threads.threadid()], state, g, s)
        else
            LightGraphs._accumulate_basic!(local_betweenness[Base.Threads.threadid()], state, g, s)
        end
    end
    betweenness = reduce(+, local_betweenness)

    LightGraphs._rescale!(betweenness,
    n_v,
    normalize,
    isdir,
    k)

    return betweenness
end

threaded_betweenness_centrality(g::AbstractGraph, k::Integer, distmx::AbstractMatrix=weights(g); normalize=true, endpoints=false) =
    threaded_betweenness_centrality(g, sample(vertices(g), k), distmx; normalize=normalize, endpoints=endpoints)


text:
""" [Tan2018](@cite) """
LifeCycleTan2018_θ_liq_ice(::Type{FT}) where {FT} =
    z -> if z <= 520.0
        FT(298.7)
    elseif z > 520.0 && z <= 1480.0
        FT(298.7) + (z - 520) * (FT(302.4) - FT(298.7)) / (1480 - 520)
    elseif z > 1480.0 && z <= 2000
        FT(302.4) + (z - 1480) * (FT(308.2) - FT(302.4)) / (2000 - 1480)
    elseif z > 2000.0
        FT(308.2) + (z - 2000) * (FT(311.85) - FT(308.2)) / (3000 - 2000)
    else
        FT(0)
    end

""" [Tan2018](@cite) """
LifeCycleTan2018_q_tot(::Type{FT}) where {FT} =
    z -> if z <= 520
        (FT(17) + z * (FT(16.3) - FT(17.0)) / 520) / 1000
    elseif z > 520.0 && z <= 1480.0
        (FT(16.3) + (z - 520) * (FT(10.7) - FT(16.3)) / (1480 - 520)) / 1000
    elseif z > 1480.0 && z <= 2000.0
        (FT(10.7) + (z - 1480) * (FT(4.2) - FT(10.7)) / (2000 - 1480)) / 1000
    elseif z > 2000.0
        (FT(4.2) + (z - 2000) * (3 - FT(4.2)) / (3000 - 2000)) / 1000
    else
        FT(0)
    end

""" [Tan2018](@cite) """
LifeCycleTan2018_u(::Type{FT}) where {FT} =
    z -> if z <= 700.0
        FT(-8.75)
    else
        FT(-8.75) + (z - 700) * (FT(-4.61) - FT(-8.75)) / (3000 - 700)
    end
""" [Tan2018](@cite) """
LifeCycleTan2018_tke(::Type{FT}) where {FT} =
    z -> if z <= 2500.0
        FT(1) - z / 3000
    else
        FT(0)
    end

# Large-scale cooling
""" [Tan2018](@cite) """
LifeCycleTan2018_dTdt(::Type{FT}) where {FT} =
    (Π, z) -> if z <= 1500.0
        FT(-2 / (3600 * 24)) * Π
    else
        FT(-2 / (3600 * 24) + (z - 1500) * (0 - -2 / (3600 * 24)) / (3000 - 1500)) * Π
    end

# geostrophic velocity profiles
""" [Tan2018](@cite) """
LifeCycleTan2018_geostrophic_u(::Type{FT}) where {FT} =
    z -> -10 + FT(1.8e-3) * z

""" [Tan2018](@cite) """
LifeCycleTan2018_geostrophic_v(::Type{FT}) where {FT} =
    z -> FT(0)

# Large-scale drying
""" [Tan2018](@cite) """
LifeCycleTan2018_dqtdt(::Type{FT}) where {FT} =
    z -> if z <= 300.0
        FT(-1.2e-8)   #kg/(kg * s)
    elseif z > 300.0 && z <= 500.0
        FT(-1.2e-8) + (z - 300) * (0 - FT(-1.2e-8)) / (500 - 300) #kg/(kg * s)
    else
        FT(0)
    end

#Large scale subsidence
""" [Tan2018](@cite) """
LifeCycleTan2018_subsidence(::Type{FT}) where {FT} =
    z -> if z <= 1500.0
        FT(0) + z * (FT(-0.65) / 100 - 0) / (1500 - 0)
    elseif z > 1500.0 && z <= 2100.0
        FT(-0.65) / 100 + (z - 1500) * (0 - FT(-0.65) / 100) / (2100 - 1500)
    else
        FT(0)
    end


text:
# This code is based on a routine originally copyright Chris Sims.
# See http://sims.princeton.edu/yftp/gensys/

"""
```
gensys(Γ0, Γ1, c, Ψ, Π)
gensys(Γ0, Γ1, c, Ψ, Π, div)
gensys(F::Base.LinAlg.GeneralizedSchur, c, Ψ, Π)
gensys(F::Base.LinAlg.GeneralizedSchur, c, Ψ, Π, div)
```

Generate state-space solution to canonical-form DSGE model.

System given as
```
Γ0*y(t) = Γ1*y(t-1) + c + Ψ*z(t) + Π*η(t),
```
with z an exogenous variable process and η being endogenously
determined one-step-ahead expectational errors.

Returned system is
```
y(t) = G1*y(t-1) + C + impact*z(t) + ywt*inv(I-fmat*inv(L))*fwt*z(t+1)
```
Returned values are
```
G1, C, impact, fmat, fwt, ywt, gev, eu, loose
```

If `z(t)` is i.i.d., the last term drops out.

If `div` is omitted from argument list, a `div`>1 is calculated.

### Return codes

* `eu[1]==1` for existence
* `eu[2]==1` for uniqueness
* `eu[1]==-1` for existence only with not-s.c. z;
* `eu==[-2,-2]` for coincident zeros.

### Notes

We constrain Julia to use the complex version of the `schurfact` routine regardless of the
types of `Γ0` and `Γ1`, to match the behavior of Matlab.  Matlab always uses the complex version
of the Schur decomposition, even if the inputs are real numbers.
"""
function gensys(Γ0, Γ1, c, Ψ, Π, args...)
    F = schurfact!(complex(Γ0), complex(Γ1))
    gensys(F, c, Ψ, Π, args...)
end

function gensys(F::Base.LinAlg.GeneralizedSchur, c, Ψ, Π)
    gensys(F, c, Ψ, Π, new_div(F))
end

# Method that does the real work. Work directly on the decomposition F
function gensys(F::Base.LinAlg.GeneralizedSchur, c, Ψ, Π, div)
    eu = [0, 0]
    ϵ = 1e-6  # small number to check convergence
    nunstab = 0
    zxz = 0
    a, b, = F[:S], F[:T]
    n = size(a, 1)

    for i in 1:n
        nunstab += (abs(b[i, i]) > div * abs(a[i,i]))
        if (abs(a[i, i]) < ϵ) && (abs(b[i, i]) < ϵ)
            zxz = 1
        end
    end

    if zxz == 1
        info("Coincident zeros.  Indeterminacy and/or nonexistence.")
        eu=[-2, -2]
        G1 = Array{Float64, 2}() ;  C = Array{Float64, 1}() ; impact = Array{Float64, 2}() ; fmat = Array{Complex{Float64}, 2}() ; fwt = Array{Complex{Float64}, 2}() ; ywt = Vector{Complex{Float64}}() ; gev = Vector{Complex{Float64}}() ; loose = Array{Float64, 2}()
        return G1, C, impact, fmat, fwt, ywt, gev, eu, loose
    end

    select = abs(F[:alpha]) .> div * abs(F[:beta])
    FS = ordschur!(F, select)
    a, b, qt, z = FS[:S], FS[:T], FS[:Q], FS[:Z]
    gev = hcat(diag(a), diag(b))
    qt1 = qt[:, 1:(n - nunstab)]
    qt2 = qt[:, (n - nunstab + 1):n]
    etawt = Ac_mul_B(qt2, Π)
    neta = size(Π, 2)

    # branch below is to handle case of no stable roots, rather than quitting with an error
    # in that case.
    if nunstab == 0
        etawt = zeros(0, neta)
        ueta = zeros(0, 0)
        deta = zeros(0, 0)
        veta = zeros(neta, 0)
        bigev = 0
    else
        etawtsvd = svdfact!(etawt)
        bigev = find(etawtsvd[:S] .> ϵ)
        ueta = etawtsvd[:U][:, bigev]
        veta = etawtsvd[:V][:, bigev]
        deta = diagm(etawtsvd[:S][bigev])
    end

    eu[1] = length(bigev) >= nunstab

    # Note that existence and uniqueness are not just matters of comparing
    # numbers of roots and numbers of endogenous errors.  These counts are
    # reported below because usually they point to the source of the problem.

    # branch below to handle case of no stable roots
    if nunstab == n
        etawt1 = zeros(0, neta)
        bigev = 0
        ueta1 = zeros(0, 0)
        veta1 = zeros(neta, 0)
        deta1 = zeros(0, 0)
    else
        etawt1 = Ac_mul_B(qt1, Π)
        ndeta1 = min(n - nunstab, neta)
        etawt1svd = svdfact!(etawt1)
        bigev = find(etawt1svd[:S] .> ϵ)
        ueta1 = etawt1svd[:U][:, bigev]
        veta1 = etawt1svd[:V][:, bigev]
        deta1 = diagm(etawt1svd[:S][bigev])
    end

    if isempty(veta1)
        unique = true
    else
        loose = veta1 - A_mul_Bc(veta, veta) * veta1
        loosesvd = svdfact!(loose)
        nloose = sum(abs(loosesvd[:S]) .> ϵ * n)
        unique = (nloose == 0)
    end

    if unique
        eu[2] = 1
    else
        info("Indeterminacy. $(nloose) loose endogeneous errors")
    end

    tmat = hcat(eye(n - nunstab), -(ueta * (deta \ veta') * veta1 * A_mul_Bc(deta1, ueta1))')

    G0 = vcat(tmat * a, hcat(zeros(nunstab, n - nunstab), eye(nunstab)))
    G1 = vcat(tmat * b, zeros(nunstab, n))

    # G0 is always non-singular because by construction there are no zeros on
    # the diagonal of a(1:n-nunstab,1:n-nunstab), which forms G0's ul corner.
    G0I = inv(G0)
    G1 = G0I * G1
    usix = (n - nunstab + 1):n
    Busix = b[usix,usix]
    Ausix = a[usix,usix]
    C = G0I * vcat(tmat * Ac_mul_B(qt, c), (Ausix - Busix) \ Ac_mul_B(qt2, c))
    impact = G0I * vcat(tmat * Ac_mul_B(qt, Ψ), zeros(nunstab, size(Ψ, 2)))
    fmat = Busix \ Ausix
    fwt = -Busix \ Ac_mul_B(qt2, Ψ)
    ywt = G0I[:, usix]

    loose = G0I * vcat(etawt1 * (eye(neta) - A_mul_Bc(veta, veta)), zeros(nunstab, neta))

    G1 = real(z * A_mul_Bc(G1, z))
    C = real(z * C)
    impact = real(z * impact)
    loose = real(z * loose)

    ywt = z * ywt

    return G1, C, impact, fmat, fwt, ywt, gev, eu, loose
end

function new_div(F::Base.LinAlg.GeneralizedSchur)
    ϵ = 1e-6  # small number to check convergence
    n = size(F[:T], 1)
    a, b = F[:S], F[:T]
    div = 1.01
    for i in 1:n
        if abs(a[i, i]) > 0
            divhat = abs(b[i, i]) / abs(a[i, i])
            if 1 + ϵ < divhat && divhat <= div
                div = .5 * (1 + divhat)
            end
        end
    end

    return div
end


text:
module TestProblems

    # using Reexport

    include("problems/harmonic_oscillator.jl")
    include("problems/kubo_oscillator.jl")
    include("problems/lotka_volterra_2d.jl")

    # @reexport using .HarmonicOscillator
    # @reexport using .KuboOscillator
    # @reexport using .LotkaVolterra2d

end


text:
# Basic decoder tests

using FountainCodes, LinearAlgebra, SparseArrays, Test

function test_decoder(A, b; expected_inactivations=nothing)
    x = transpose(A)*b
    tc = TestConstraints(A, x)
    decoder = Decoder(A)
    dec = decode(A, tc; decoder)
    if eltype(A) <: AbstractFloat
        @test dec ≈ b
    else
        @test dec == b
    end
    if !isnothing(expected_inactivations)
        @test decoder.num_inactivated == expected_inactivations
    end
    true
end

"""Test decoding with a diagonal constraint matrix."""
function test_diagonal(K::Integer; Tv=GF256)
    rng = MersenneTwister(123)
    constraints = [sparsevec([i], [rand_nonzero(rng, Tv)], K) for i in 1:K]
    A = hcat(constraints...)
    b = rand(rng, Tv, K)
    test_decoder(A, b, expected_inactivations=0)
end
for K in [1, 10, 100, 200, 250, 254]
    test_diagonal(K)
end

function test_bidiagonal(K::Integer; Tv=GF256, Ti=Int, permuted=false)
    rng = MersenneTwister(123)
    dv = rand_nonzero(rng, Tv, K)
    ev = rand_nonzero(rng, Tv, K-1)
    b = rand(rng, Tv, K)
    if permuted
        p = randperm(rng, K)
        q = randperm(rng, K)
        A = SparseMatrixCSC{Tv,Ti}(sparse(Bidiagonal(dv, ev, :U)[p, q]))
    else
        A = SparseMatrixCSC{Tv,Ti}(sparse(Bidiagonal(dv, ev, :U)))
    end
    test_decoder(A, b, expected_inactivations=0)
end
for K in [10, 100, 200, 250, 254]
    test_bidiagonal(K, permuted=false)
    test_bidiagonal(K, permuted=true)
end
test_bidiagonal(10, Ti=Int32, permuted=false)

function test_tridiagonal(K::Integer; Tv=GF256, permuted=false)
    rng = MersenneTwister(123)
    du = rand_nonzero(rng, Tv, K-1)
    d = rand_nonzero(rng, Tv, K)
    dl = rand_nonzero(rng, Tv, K-1)
    b = rand(rng, Tv, K)
    if permuted
        p = randperm(rng, K)
        q = randperm(rng, K)
        A = sparse(Tridiagonal(dl, d, du)[p, q])
    else
        A = sparse(Tridiagonal(dl, d, du))
    end
    test_decoder(A, b, expected_inactivations=1)
end
for K in [10, 100, 200, 250, 254]
    test_tridiagonal(K, permuted=false)
    test_tridiagonal(K, permuted=true)
end
test_tridiagonal(10, Tv=Float64, permuted=false)

"""Test decoding for a dense constraint matrix."""
function test_dense(m::Integer, n::Integer=m; Tv=GF256)
    rng = MersenneTwister(123)
    A = sparse(rand_nonzero(rng, Tv, m, n))
    b = rand(rng, Tv, m)
    test_decoder(A, b, expected_inactivations=m-1)
end
for K in [10, 100, 200, 250, 254]
    test_dense(K)
end

"""Test that connected components are selected correctly."""
function test_inactivations_1(Tv=GF256)
    rng = MersenneTwister(123)
    Is = [
        (1, 1), (1, 2),
        (2, 2), (2, 3),
        (3, 3), (3, 4),
        (4, 4), (4, 5),
        (5, 1), (5, 2),
        (6, 2), (6, 3),
        (7, 4), (7, 5)
    ]
    M = zeros(Tv, 7, 5)
    for (i, j) in Is
        M[i, j] = rand_nonzero(rng, Tv)
    end
    b = rand(rng, Tv, 5)
    p = randperm(rng, size(M, 1))
    q = randperm(rng, size(M, 2))
    A = sparse(M[p, q]')
    test_decoder(A, b, expected_inactivations=1)
end
test_inactivations_1()

"""Test that connected components are selected correctly."""
function test_inactivations_2(Tv=GF256)
    rng = MersenneTwister(123)
    Is = [
        # Component 1 (4 symbols)
        (1, 1), (1, 2),
        (2, 2), (2, 3),
        (3, 3), (3, 4),
        # Component 2 (3 symbols)
        (4, 5), (4, 6),
        (5, 6), (5, 7),
    ]
    M = zeros(Tv, 8, 7)
    for (i, j) in Is
        M[i, j] = rand_nonzero(rng, Tv)
    end
    M[6, 1:4] .= rand_nonzero(rng, Tv, 4)
    M[7, 1:4] .= rand_nonzero(rng, Tv, 4)
    M[8, 1:4] .= rand_nonzero(rng, Tv, 4)
    M[6, 5] = rand_nonzero(rng, Tv)
    M[7, 6] = rand_nonzero(rng, Tv)
    M[8, 7] = rand_nonzero(rng, Tv)
    b = rand(rng, Tv, 7)
    p = randperm(rng, size(M, 1))
    q = randperm(rng, size(M, 2))
    A = sparse(M[p, q]')
    test_decoder(A, b, expected_inactivations=1)
end
test_inactivations_2()

max_stars_repo_name: severinson/RaptorCodes

max_issues_repo_name: severinson/RaptorCodes

max_forks_repo_name: severinson/RaptorCodes


text:
# Matrix: create matrix representation of LinearMap
function Base.Matrix(A::LinearMap)
    M, N = size(A)
    T = eltype(A)
    mat = Matrix{T}(undef, (M, N))
    v = fill(zero(T), N)
    @inbounds for i = 1:N
        v[i] = one(T)
        mul!(view(mat, :, i), A, v)
        v[i] = zero(T)
    end
    return mat
end
Base.Array(A::LinearMap) = Matrix(A)
Base.convert(::Type{Matrix}, A::LinearMap) = Matrix(A)
Base.convert(::Type{Array}, A::LinearMap) = convert(Matrix, A)
Base.convert(::Type{AbstractMatrix}, A::LinearMap) = convert(Matrix, A)
Base.convert(::Type{AbstractArray}, A::LinearMap) = convert(AbstractMatrix, A)

# special cases
Base.convert(::Type{AbstractMatrix}, A::WrappedMap) = convert(AbstractMatrix, A.lmap)
Base.convert(::Type{Matrix}, A::WrappedMap) = convert(Matrix, A.lmap)
function Base.convert(::Type{Matrix}, ΣA::LinearCombination{<:Any,<:Tuple{Vararg{MatrixMap}}})
    if length(ΣA.maps) <= 10
        return (+).(map(A->getfield(A, :lmap), ΣA.maps)...)
    else
        S = zero(first(ΣA.maps).lmap)
        for A in ΣA.maps
            S .+= A.lmap
        end
        return S
    end
end
function Base.convert(::Type{Matrix}, AB::CompositeMap{<:Any,<:Tuple{MatrixMap,MatrixMap}})
    B, A = AB.maps
    return A.lmap*B.lmap
end
function Base.convert(::Type{Matrix}, λA::CompositeMap{<:Any,<:Tuple{MatrixMap,UniformScalingMap}})
    A, J = λA.maps
    return J.λ*A.lmap
end
function Base.convert(::Type{Matrix}, Aλ::CompositeMap{<:Any,<:Tuple{UniformScalingMap,MatrixMap}})
    J, A = Aλ.maps
    return A.lmap*J.λ
end

Base.Matrix(A::BlockMap) = materialize!(Matrix{eltype(A)}(undef, size(A)), A)

# sparse: create sparse matrix representation of LinearMap
function SparseArrays.sparse(A::LinearMap{T}) where {T}
    M, N = size(A)
    rowind = Int[]
    nzval = T[]
    colptr = Vector{Int}(undef, N+1)
    v = fill(zero(T), N)
    Av = Vector{T}(undef, M)

    for i = 1:N
        v[i] = one(T)
        mul!(Av, A, v)
        js = findall(!iszero, Av)
        colptr[i] = length(nzval) + 1
        if length(js) > 0
            append!(rowind, js)
            append!(nzval, Av[js])
        end
        v[i] = zero(T)
    end
    colptr[N+1] = length(nzval) + 1

    return SparseMatrixCSC(M, N, colptr, rowind, nzval)
end

Base.convert(::Type{SparseMatrixCSC}, A::LinearMap) = sparse(A)
Base.convert(::Type{SparseMatrixCSC}, A::WrappedMap) = convert(SparseMatrixCSC, A.lmap)
Base.convert(::Type{SparseMatrixCSC}, A::BlockMap) = materialize!(spzeros(eltype(A), size(A)...), A)


text:
#
# Computes the total potential energy of the system and
# the force on every particle. Modifies input vector f
#
function energyandforce!(p,f,data)
  utotal = 0.
  for v in f
    v .= 0.
  end
  for ip in 1:data.N-1
    for jp in ip+1:data.N
      wrap!(p[ip],p[jp],data.side)
      rij = sqrt( (p[ip][1]-p[jp][1])^2 + (p[ip][2]-p[jp][2])^2 )
      if rij < data.cutoff
        upair, fpair = forcepair(p[ip],p[jp],rij,data)
        utotal += upair
        f[ip] .= f[ip] .+ fpair
        f[jp] .= f[jp] .- fpair
      end
    end
  end
  return utotal, f
end


text:
@testset "Testing Cartesian Sequence submodule" begin

fnSMTD = joinpath(datadir,"mdf","systemMatrixCartesian.mdf")
fnSMFDMP = joinpath(tmpdir,"mdf","systemMatrixCartesianMP.mdf")
fnSMFDSP = joinpath(tmpdir,"mdf","systemMatrixCartesianSP.mdf")

smTD = MPIFile(fnSMTD)
@test typeof(smTD) <: MDFFileV2

@test acqNumPeriodsPerFrame(smTD) == 6500
@test rxNumSamplingPoints(smTD) == 76
@test size(getSystemMatrix(smTD,1:10)) == (81,6500*10)
@test size(getMeasurements(smTD)) == (76, 2, 6500, 81)
@test size(getMeasurements(smTD, numPeriodAverages=65)) == (76, 2, 100, 81)
@test size(getMeasurements(smTD, numPeriodAverages=65, numPeriodGrouping=100)) == (7600, 2, 1, 81)
@test size(getMeasurementsFD(smTD, frequencies=1:10)) == (10, 6500, 81)

saveasMDF(fnSMFDMP, smTD, numPeriodAverages=65, applyCalibPostprocessing=true)
smFDMP = MPIFile(fnSMFDMP)

@test acqNumPeriodsPerFrame(smFDMP) == 100
@test rxNumSamplingPoints(smFDMP) == 76
@test size(getSystemMatrix(smFDMP,1:10)) == (81,10*100)

saveasMDF(fnSMFDSP, smTD, numPeriodAverages=65, applyCalibPostprocessing=true, numPeriodGrouping=100)
smFDSP = MPIFile(fnSMFDSP)

@test acqNumPeriodsPerFrame(smFDSP) == 1
@test rxNumSamplingPoints(smFDSP) == 7600
@test size(getSystemMatrix(smFDSP,1:10)) == (81,10)

end


text:
module HypercubeTransform

using Requires: @require
import Distributions
const Dists = Distributions
using Distributions: quantile
import MeasureTheory
const MT = MeasureTheory
using MLStyle
using GeneralizedGenerated
using Tricks: static_hasmethod
using DocStringExtensions
import TransformVariables: as, transform, dimension, ∞

export transform, dimension, ascube, asflat, transform_and_logjac, transform_logdensity

include("utility.jl")
include("transform.jl")
include("composite.jl")
include("asflat.jl")

#function __init__()
 #   @require Soss="8ce77f84-9b61-11e8-39ff-d17a774bf41c" include("soss.jl")
#end

end #module


text:
using FactCheck
using LabelMe

facts("annotation") do
    context("empty") do
        expected = Annotation()
        element = XML.element(expected)
        actual = XML.annotation(element)
        @fact actual --> expected
    end
    context("non-empty") do
        expected = Annotation("img1.xml", "test", Source(), nothing, [Object()])
        element = XML.element(expected)
        actual = XML.annotation(element)
        @fact actual --> expected
    end
end

facts("source") do
    context("empty") do
        expected = Source()
        element = XML.element(expected)
        actual = XML.source(element)
        @fact actual --> expected
    end
    context("non-empty") do
        expected = Source("image source", "annotation source")
        element = XML.element(expected)
        actual = XML.source(element)
        @fact actual --> expected
    end
end

facts("object") do
    context("empty") do
        expected = Object()
        element = XML.element(expected)
        actual = XML.object(element)
        @fact actual --> expected
    end
    context("non-empty") do
        expected = Object(1, "car", nothing, true, false, "colour:white", 0, [2,3], DateTime(2013,7,1,12,30,59), Polygon())
        element = XML.element(expected)
        actual = XML.object(element)
        @fact actual --> expected
    end
end

facts("polygon") do
    context("empty") do
        expected = Polygon()
        polygon_element = XML.element(expected)
        actual = XML.polygon(polygon_element)
        @fact actual --> expected
    end
    context("username") do
        expected = Polygon("bob", [])
        polygon_element = XML.element(expected)
        actual = XML.polygon(polygon_element)
        @fact actual --> expected
    end
    context("points") do
        expected = Polygon("", [(2,3),(5,8)])
        polygon_element = XML.element(expected)
        actual = XML.polygon(polygon_element)
        @fact actual --> expected
    end
end


text:
# Note:
# Another Possible poisson generator:
# https://github.com/JuliaStats/Distributions.jl/blob/master/src/univariate/discrete/poisson.jl

# Note on randjump():
# https://www.juliabloggers.com/basics-of-generating-random-numbers-in-julia/

using Random
using Future

# The output of the stream can connect 1 or more Connections.
mutable struct PoissonStream <: AbstractBitStream
    base::BaseData{UInt8}
    seed::Int64

    # The Interspike interval (ISI) counter is populated by a value.
    # When the counter reaches 0 a spike is placed on the output
    # for single pass.
    isi::UInt64

    # This equal to lamba rates.
    firing_rate::Float64

    rng::MersenneTwister

    function PoissonStream(seed::UInt64, firing_rate::Float64 = 0.002)
        o = new()
        o.base = BaseData{UInt8}()
        o.firing_rate = firing_rate
        o.seed = seed
        o.isi = 0
        reset!(o)
        step!(o)
        o
    end
end

# Create an event per interval of time, for example, spikes in a 1 sec span.
# A firing rate given in rate/ms, for example, 0.2 in 1ms (0.2/1)
# or 200 in 1sec (200/1000ms)
function next!(stream::PoissonStream)
    p = -log(1.0 - rand(stream.rng, 1)[1]) / stream.firing_rate
    UInt64(round(p))
end

function reset!(stream::PoissonStream)
    stream.rng = Future.randjump(MersenneTwister(stream.seed), big(10)^20)
    stream.isi = next!(stream)
end

function step!(stream::PoissonStream)
    # Check ISI counter
   	if stream.isi == 0
	    # Time to generate a spike
        stream.base.output = 1
        stream.isi = next!(stream)
    else
        stream.base.output = 0
        stream.isi -= 1
    end
end

function output(stream::PoissonStream)
    stream.base.output & 0x01
end

function output(stream::PoissonStream, id::Int64)
    stream.base.output & 0x01
end


text:
# SPDX-License-Identifier: X11
# 2020-11-23
# Quizzes (200pt)

function main()
  N,X = parse.(Int, (split ∘ readline)())
  S = readline()

  for c ∈ S
    if c == 'o'
      X += 1
    else
      if X > 0
        X -= 1
      end
    end
  end

  println(X)
end

main()

max_stars_repo_name: jthvai/competitive-programming

max_issues_repo_name: jthvai/competitive-programming

max_forks_repo_name: jthvai/competitive-programming


text:
# This file was generated, do not modify it. # hide
TN = down_down = sum(ŷ .== y .== "Down")
FN = down_up = sum(ŷ .!= y .== "Up")
FP = up_down = sum(ŷ .!= y .== "Down")
TP = up_up = sum(ŷ .== y .== "Up")

conf_mat = [down_down down_up; up_down up_up]

max_stars_repo_name: vdayanand/MLJTutorials

max_issues_repo_name: vdayanand/MLJTutorials

max_forks_repo_name: vdayanand/MLJTutorials


text:
export BPFCmd, BPFMapType, BPFProgType, BPFAttachType, BPFHelper

@enum BPFCmd begin
    BPF_MAP_CREATE
    BPF_MAP_LOOKUP_ELEM
    BPF_MAP_UPDATE_ELEM
    BPF_MAP_DELETE_ELEM
    BPF_MAP_GET_NEXT_KEY
    BPF_PROG_LOAD
    BPF_OBJ_PIN
    BPF_OBJ_GET
    BPF_PROG_ATTACH
    BPF_PROG_DETACH
    BPF_PROG_TEST_RUN
    BPF_PROG_GET_NEXT_ID
    BPF_MAP_GET_NEXT_ID
    BPF_PROG_GET_FD_BY_ID
    BPF_MAP_GET_FD_BY_ID
    BPF_OBJ_GET_INFO_BY_FD
    BPF_PROG_QUERY
    BPF_RAW_TRACEPOINT_OPEN
    BPF_BTF_LOAD
    BPF_BTF_GET_FD_BY_ID
    BPF_TASK_FD_QUERY
    BPF_MAP_LOOKUP_AND_DELETE_ELEM
    BPF_MAP_FREEZE
    BPF_BTF_GET_NEXT_ID
    BPF_MAP_LOOKUP_BATCH
    BPF_MAP_LOOKUP_AND_DELETE_BATCH
    BPF_MAP_UPDATE_BATCH
    BPF_MAP_DELETE_BATCH
    BPF_LINK_CREATE
    BPF_LINK_UPDATE
end

@enum BPFMapType begin
    BPF_MAP_TYPE_UNSPEC
    BPF_MAP_TYPE_HASH
    BPF_MAP_TYPE_ARRAY
    BPF_MAP_TYPE_PROG_ARRAY
    BPF_MAP_TYPE_PERF_EVENT_ARRAY
    BPF_MAP_TYPE_PERCPU_HASH
    BPF_MAP_TYPE_PERCPU_ARRAY
    BPF_MAP_TYPE_STACK_TRACE
    BPF_MAP_TYPE_CGROUP_ARRAY
    BPF_MAP_TYPE_LRU_HASH
    BPF_MAP_TYPE_LRU_PERCPU_HASH
    BPF_MAP_TYPE_LPM_TRIE
    BPF_MAP_TYPE_ARRAY_OF_MAPS
    BPF_MAP_TYPE_HASH_OF_MAPS
    BPF_MAP_TYPE_DEVMAP
    BPF_MAP_TYPE_SOCKMAP
    BPF_MAP_TYPE_CPUMAP
    BPF_MAP_TYPE_XSKMAP
    BPF_MAP_TYPE_SOCKHASH
    BPF_MAP_TYPE_CGROUP_STORAGE
    BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
    BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
    BPF_MAP_TYPE_QUEUE
    BPF_MAP_TYPE_STACK
    BPF_MAP_TYPE_SK_STORAGE
    BPF_MAP_TYPE_DEVMAP_HASH
    BPF_MAP_TYPE_STRUCT_OPS
end

@enum BPFProgType begin
    BPF_PROG_TYPE_UNSPEC
    BPF_PROG_TYPE_SOCKET_FILTER
    BPF_PROG_TYPE_KPROBE
    BPF_PROG_TYPE_SCHED_CLS
    BPF_PROG_TYPE_SCHED_ACT
    BPF_PROG_TYPE_TRACEPOINT
    BPF_PROG_TYPE_XDP
    BPF_PROG_TYPE_PERF_EVENT
    BPF_PROG_TYPE_CGROUP_SKB
    BPF_PROG_TYPE_CGROUP_SOCK
    BPF_PROG_TYPE_LWT_IN
    BPF_PROG_TYPE_LWT_OUT
    BPF_PROG_TYPE_LWT_XMIT
    BPF_PROG_TYPE_SOCK_OPS
    BPF_PROG_TYPE_SK_SKB
    BPF_PROG_TYPE_CGROUP_DEVICE
    BPF_PROG_TYPE_SK_MSG
    BPF_PROG_TYPE_RAW_TRACEPOINT
    BPF_PROG_TYPE_CGROUP_SOCK_ADDR
    BPF_PROG_TYPE_LWT_SEG6LOCAL
    BPF_PROG_TYPE_LIRC_MODE2
    BPF_PROG_TYPE_SK_REUSEPORT
    BPF_PROG_TYPE_FLOW_DISSECTOR
    BPF_PROG_TYPE_CGROUP_SYSCTL
    BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE
    BPF_PROG_TYPE_CGROUP_SOCKOPT
    BPF_PROG_TYPE_TRACING
    BPF_PROG_TYPE_STRUCT_OPS
    BPF_PROG_TYPE_EXT
    BPF_PROG_TYPE_LSM
end

@enum BPFAttachType begin
    BPF_CGROUP_INET_INGRESS
    BPF_CGROUP_INET_EGRESS
    BPF_CGROUP_INET_SOCK_CREATE
    BPF_CGROUP_SOCK_OPS
    BPF_SK_SKB_STREAM_PARSER
    BPF_SK_SKB_STREAM_VERDICT
    BPF_CGROUP_DEVICE
    BPF_SK_MSG_VERDICT
    BPF_CGROUP_INET4_BIND
    BPF_CGROUP_INET6_BIND
    BPF_CGROUP_INET4_CONNECT
    BPF_CGROUP_INET6_CONNECT
    BPF_CGROUP_INET4_POST_BIND
    BPF_CGROUP_INET6_POST_BIND
    BPF_CGROUP_UDP4_SENDMSG
    BPF_CGROUP_UDP6_SENDMSG
    BPF_LIRC_MODE2
    BPF_FLOW_DISSECTOR
    BPF_CGROUP_SYSCTL
    BPF_CGROUP_UDP4_RECVMSG
    BPF_CGROUP_UDP6_RECVMSG
    BPF_CGROUP_GETSOCKOPT
    BPF_CGROUP_SETSOCKOPT
    BPF_TRACE_RAW_TP
    BPF_TRACE_FENTRY
    BPF_TRACE_FEXIT
    BPF_MODIFY_RETURN
    BPF_LSM_MAC
end

@enum BPFHelper begin
    unspec
    map_lookup_elem
    map_update_elem
    map_delete_elem
    probe_read
    ktime_get_ns
    trace_printk
    get_prandom_u32
    get_smp_processor_id
    skb_store_bytes
    l3_csum_replace
    l4_csum_replace
    tail_call
    clone_redirect
    get_current_pid_tgid
    get_current_uid_gid
    get_current_comm
    get_cgroup_classid
    skb_vlan_push
    skb_vlan_pop
    skb_get_tunnel_key
    skb_set_tunnel_key
    perf_event_read
    redirect
    get_route_realm
    perf_event_output
    skb_load_bytes
    get_stackid
    csum_diff
    skb_get_tunnel_opt
    skb_set_tunnel_opt
    skb_change_proto
    skb_change_type
    skb_under_cgroup
    get_hash_recalc
    get_current_task
    probe_write_user
    current_task_under_cgroup
    skb_change_tail
    skb_pull_data
    csum_update
    set_hash_invalid
    get_numa_node_id
    skb_change_head
    xdp_adjust_head
    probe_read_str
    get_socket_cookie
    get_socket_uid
    set_hash
    setsockopt
    skb_adjust_room
    redirect_map
    sk_redirect_map
    sock_map_update
    xdp_adjust_meta
    perf_event_read_value
    perf_prog_read_value
    getsockopt
    override_return
    sock_ops_cb_flags_set
    msg_redirect_map
    msg_apply_bytes
    msg_cork_bytes
    msg_pull_data
    bind
    xdp_adjust_tail
    skb_get_xfrm_state
    get_stack
    skb_load_bytes_relative
    fib_lookup
    sock_hash_update
    msg_redirect_hash
    sk_redirect_hash
    lwt_push_encap
    lwt_seg6_store_bytes
    lwt_seg6_adjust_srh
    lwt_seg6_action
    rc_repeat
    rc_keydown
    skb_cgroup_id
    get_current_cgroup_id
    get_local_storage
    sk_select_reuseport
    skb_ancestor_cgroup_id
    sk_lookup_tcp
    sk_lookup_udp
    sk_release
    map_push_elem
    map_pop_elem
    map_peek_elem
    msg_push_data
    msg_pop_data
    rc_pointer_rel
    spin_lock
    spin_unlock
    sk_fullsock
    tcp_sock
    skb_ecn_set_ce
    get_listener_sock
    skc_lookup_tcp
    tcp_check_syncookie
    sysctl_get_name
    sysctl_get_current_value
    sysctl_get_new_value
    sysctl_set_new_value
    strtol
    strtoul
    sk_storage_get
    sk_storage_delete
    send_signal
    tcp_gen_syncookie
    skb_output
    probe_read_user
    probe_read_kernel
    probe_read_user_str
    probe_read_kernel_str
    tcp_send_ack
    send_signal_thread
    jiffies64
    read_branch_records
    get_ns_current_pid_tgid
    xdp_output
    get_netns_cookie
    get_current_ancestor_cgroup_id
    sk_assign
end

const PERF_MAX_STACK_DEPTH = 127

# Kernel structures

if has_vmlinux

const pt_regs = c".VMLinux.struct pt_regs"
pointertype(::Type{pt_regs}) = Cptr{pt_regs}

const xdp_md = c".VMLinux.struct xdp_md"
pointertype(::Type{xdp_md}) = Cptr{xdp_md}

const sk_buff = c".VMLinux.struct sk_buff"
pointertype(::Type{sk_buff}) = Cptr{sk_buff}

else

## Perf/Ptrace

# TODO: pt_regs for other architectures
struct pt_regs
    ebx::Clong
    ecx::Clong
    edx::Clong
    esi::Clong
    edi::Clong
    ebp::Clong
    eax::Clong
    xds::Cint
    xes::Cint
    xfs::Cint
    xgs::Cint
    orig_eax::Clong
    eip::Clong
    xcs::Cint
    eflags::Clong
    esp::Clong
    xss::Cint
end
pointertype(::Type{pt_regs}) = Ptr{pt_regs}

## XDP

struct xdp_md
    data::UInt32
    data_end::UInt32
    data_meta::UInt32
    ingress_ifindex::UInt32
    rx_queue_index::UInt32
end
pointertype(::Type{xdp_md}) = Ptr{xdp_md}

## Sockets/TC

struct sk_buff
    len::UInt32
    pkt_type::UInt32
    mark::UInt32
    queue_mapping::UInt32
    protocol::UInt32
    vlan_present::UInt32
    vlan_tci::UInt32
    vlan_proto::UInt32
    priority::UInt32
    ingress_ifindex::UInt32
    ifindex::UInt32
    tc_index::UInt32
    cb::NTuple{5,UInt32}
    hash::UInt32
    tc_classid::UInt32
    data::UInt32
    data_end::UInt32
    napi_id::UInt32

    family::UInt32
    remote_ip4::UInt32           # Stored in network byte order
    local_ip4::UInt32            # Stored in network byte order
    remote_ip6::NTuple{4,UInt32} # Stored in network byte order
    local_ip6::NTuple{4,UInt32}  # Stored in network byte order
    remote_port::UInt32          # Stored in network byte order
    local_port::UInt32           # Stored in host byte order

    data_meta::UInt32
    flow_keys::Ptr{Cvoid} #__bpf_md_ptr(struct bpf_flow_keys *, flow_keys);
    tstamp::UInt64
    wire_len::UInt32
    gso_segs::UInt32
    sk::Ptr{Cvoid} #__bpf_md_ptr(struct bpf_sock *, sk);
    gso_size::UInt32
end
pointertype(::Type{sk_buff}) = Ptr{sk_buff}

end


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    HalfEdge(head, elem, prev, next, half)

Stores the indices of the `head` vertex, the `prev`
and `next` edges in the left `elem`, and the opposite
`half`-edge. For some half-edges the `elem` may be
`nothing`, e.g. border edges of the mesh.

See [`HalfEdgeTopology`](@ref) for more details.
"""
mutable struct HalfEdge
  head::Int
  elem::Union{Int,Nothing}
  prev::HalfEdge
  next::HalfEdge
  half::HalfEdge
  HalfEdge(head, elem) = new(head, elem)
end

"""
    loop(halfedge)

Loop over the heads of a cycle starting at given `halfedge`.
"""
function loop(e::HalfEdge)
  n = e.next
  v = [e.head]
  while n != e
    push!(v, n.head)
    n = n.next
  end
  v
end

function Base.show(io::IO, e::HalfEdge)
  print(io, "HalfEdge($(e.head), $(e.elem))")
end

"""
    HalfEdgeTopology(elements)
    HalfEdgeTopology(halfedges)

A data structure for orientable 2-manifolds based on
half-edges constructed from a vector of connectivity
`elements` or from a vector of pairs of `halfedges`.

## Example

Construct half-edge topology from a list of top-faces:

```julia
elements = connect.([(1,2,3),(3,2,4,5)])
topology = HalfEdgeTopology(elements)
```

See also [`Topology`](@ref).

## References

* Kettner, L. (1999). [Using generic programming for
  designing a data structure for polyhedral surfaces]
  (https://www.sciencedirect.com/science/article/pii/S0925772199000073)

### Notes

- Two types of half-edges exist (Kettner 1999). This
  implementation is the most common type that splits
  the incident elements.

- A vector of `halfedges` together with a dictionary of
  `half4elem` and a dictionary of `half4vert` can be
  used to retrieve topolological relations in optimal
  time. In this case, `half4vert[i]` returns the index
  of the half-edge in `halfedges` with head equal to `i`.
  Similarly, `half4elem[i]` returns the index of a
  half-edge in `halfedges` that is in the element `i`.
  Additionally, a dictionary `edge4pair` returns the
  index of the edge (i.e. two halves) for a given
  pair of vertices.
"""
struct HalfEdgeTopology <: Topology
  halfedges::Vector{HalfEdge}
  half4elem::Dict{Int,Int}
  half4vert::Dict{Int,Int}
  edge4pair::Dict{Tuple{Int,Int},Int}
end

function HalfEdgeTopology(halves::AbstractVector{Tuple{HalfEdge,HalfEdge}})
  # make sure that first half-edge is in the interior
  ordered = [isnothing(h₁.elem) ? (h₂, h₁) : (h₁, h₂) for (h₁, h₂) in halves]

  # flatten pairs of half-edges into a vector
  halfedges = [half for pair in ordered for half in pair]

  # map element and vertex to a half-edge
  half4elem = Dict{Int,Int}()
  half4vert = Dict{Int,Int}()
  for (i, h) in enumerate(halfedges)
    if !isnothing(h.elem) # interior half-edge
      if !haskey(half4elem, h.elem)
        half4elem[h.elem] = i
      end
      if !haskey(half4vert, h.head)
        half4vert[h.head] = i
      end
    end
  end

  # map pair of vertices to an edge (i.e. two halves)
  edge4pair = Dict{Tuple{Int,Int},Int}()
  for (i, (h₁, h₂)) in enumerate(ordered)
    u, v = h₁.head, h₂.head
    edge4pair[(u, v)] = i
    edge4pair[(v, u)] = i
  end

  HalfEdgeTopology(halfedges, half4elem, half4vert, edge4pair)
end

function HalfEdgeTopology(elems::AbstractVector{<:Connectivity})
  @assert all(e -> paramdim(e) == 2, elems) "invalid element for half-edge topology"

  # number of vertices is the maximum index in connectivities
  nvertices = maximum(i for e in elems for i in indices(e))

  # initialization step
  half4pair = Dict{Tuple{Int,Int},HalfEdge}()
  for (e, elem) in Iterators.enumerate(elems)
    inds = collect(indices(elem))
    v = CircularVector(inds)
    n = length(v)
    for i in 1:n
      half4pair[(v[i], v[i+1])] = HalfEdge(v[i], e)
    end
  end

  # add missing pointers
  for elem in elems
    inds = collect(indices(elem))
    v = CircularVector(inds)
    n = length(v)
    for i in 1:n
      # update pointers prev and next
      he = half4pair[(v[i], v[i+1])]
      he.prev = half4pair[(v[i-1],   v[i])]
      he.next = half4pair[(v[i+1], v[i+2])]

      # if not a border element, update half
      if haskey(half4pair, (v[i+1], v[i]))
        he.half = half4pair[(v[i+1], v[i])]
      else # create half-edge for border
        be = HalfEdge(v[i+1], nothing)
        be.half = he
        he.half = be
      end
    end
  end

  # save halfedges in a vector of pairs
  halves  = Vector{Tuple{HalfEdge,HalfEdge}}()
  visited = Set{Tuple{Int,Int}}()
  for ((u, v), he) in half4pair
    if (u, v) ∉ visited
      push!(halves,  (he, he.half))
      push!(visited, (u, v))
      push!(visited, (v, u))
    end
  end

  HalfEdgeTopology(halves)
end

"""
    half4elem(e, t)

Return a half-edge of the half-edge topology `t` on the `e`-th elem.
"""
half4elem(e::Integer, t::HalfEdgeTopology) = t.halfedges[t.half4elem[e]]

"""
    half4vert(v, t)

Return the half-edge of the half-edge topology `t` for which the
head is the `v`-th index.
"""
half4vert(v::Integer, t::HalfEdgeTopology) = t.halfedges[t.half4vert[v]]

"""
    half4edge(e, t)

Return the half-edge of the half-edge topology `t` for the edge `e`.

### Notes

Always return the half-edge to the "left".
"""
half4edge(e::Integer, t::HalfEdgeTopology) = t.halfedges[2e - 1]

"""
    half4pair(uv, t)

Return the half-edge of the half-edge topology `t` for the pair of
vertices `uv`.

### Notes

Always return the half-edge to the "left".
"""
half4pair(uv::Tuple{Int,Int}, t::HalfEdgeTopology) = half4edge(t.edge4pair[uv], t)

"""
    edge4pair(uv, t)

Return the edge of the half-edge topology `t` for the pair of vertices `uv`.
"""
edge4pair(uv, t) = t.edge4pair[uv]

# ---------------------
# HIGH-LEVEL INTERFACE
# ---------------------

nvertices(t::HalfEdgeTopology) = length(t.half4vert)

function faces(t::HalfEdgeTopology, rank)
  if rank == 2
    elements(t)
  elseif rank == 1
    facets(t)
  else
    throw(ArgumentError("invalid rank for half-edge topology"))
  end
end

function element(t::HalfEdgeTopology, ind)
  v = loop(half4elem(ind, t))
  connect(Tuple(v))
end

nelements(t::HalfEdgeTopology) = length(t.half4elem)

function facet(t::HalfEdgeTopology, ind)
  e = half4edge(ind, t)
  connect((e.head, e.half.head))
end

nfacets(t::HalfEdgeTopology) = length(t.halfedges) ÷ 2

# ------------
# CONVERSIONS
# ------------

Base.convert(::Type{<:HalfEdgeTopology}, t::Topology) =
  HalfEdgeTopology(collect(elements(t)))


text:
using LightQuery: @>, By, Group, @name, make_columns, over, row_info, Rows,
    value
using CSV: File
import CSV
using DataFrames: by, DataFrame
using Tables: schema
using BenchmarkTools: @btime

cd("/home/brandon/perf")

filename =

file = File("Performance_2000Q1.txt_0",
    delim = '|',
    header = Symbol.(string.("Column", 1:31)),
    missingstrings = ["NULL", ""],
    dateformat = "mm/dd/yyyy",
    truestrings = ["Y"],
    falsestrings = ["N"]
)

function process_with_lightquery(file)
    Line = row_info(schema(file))

    @name @> file |>
    over(_, @_ (:Column1,)(Line(_))) |>
    make_columns |>
    Rows |>
    Group(By(_, :Column1)) |>
    over(_, @_ (Count = length(value(_)),)) |>
    make_columns
end

@btime process_with_lightquery(file)

function process_with_dataframes_meta(file)
    by(DataFrame(file), :Column1, :Column1 => length)
end

@btime process_with_dataframes_meta(file)