text:
using ImageShow, Colors, FixedPointNumbers, FileIO, OffsetArrays, PaddedViews
import ImageCore: colorview, normedview
# We jump through some hoops so that this test script may work
# whether or not ImageTransformations (a fortiori Images) is loaded.
# See below for details.

# don't import ImageTransformations: restrict

using Test

const workdir = joinpath(tempdir(), "Images")
if !isdir(workdir)
    mkdir(workdir)
end

@testset "show (MIME)" begin
    # Test that we remembered to turn off Colors.jl's colorswatch display
    @test !showable(MIME("image/svg+xml"), rand(Gray{N0f8}, 5, 5))
    @test !showable(MIME("image/svg+xml"), rand(RGB{N0f8},  5, 5))
    @test showable(MIME("image/png"), rand(Gray{N0f8}, 5, 5))
    @test showable(MIME("image/png"), rand(RGB{N0f8},  5, 5))
    @testset "no compression or expansion" begin
        A = N0f8[0.01 0.99; 0.25 0.75]
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            show(file, MIME("image/png"), Gray.(A), minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test b == A

        img = fill(RGB{N0f16}(1,0,0), 1, 1)
        open(fn, "w") do file
            show(IOContext(file, :full_fidelity=>true), MIME("image/png"), img, minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test b == img

        A = N0f8[0.01 0.99; 0.25 0.75]
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            show(IOContext(file, :full_fidelity=>true), MIME("image/png"), Gray.(A), minpixels=5, maxpixels=typemax(Int))
        end
        @test load(fn) == A

        A = N0f8[0.01 0.4 0.99; 0.25 0.8 0.75; 0.6 0.2 0.0]
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            show(IOContext(file, :full_fidelity=>true), MIME("image/png"), Gray.(A), minpixels=0, maxpixels=5)
        end
        @test load(fn) == A

        # a genuinely big image (tests the defaults)
        abig = colorview(Gray, normedview(rand(UInt8, 1024, 1023)))
        fn = joinpath(workdir, "big.png")
        open(fn, "w") do file
            show(IOContext(file, :full_fidelity=>true), MIME("image/png"), abig, maxpixels=10^6)
        end
        b = load(fn)
        @test b == abig
    end
    @testset "colorspace normalization" begin
        img = fill(HSV{Float64}(0.5, 0.5, 0.5), 1, 1)
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            show(file, MIME("image/png"), img, minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test b == convert(Array{RGB{N0f8}}, img)
        img = fill(RGB{N0f16}(1,0,0), 1, 1)
        open(fn, "w") do file
            show(file, MIME("image/png"), img, minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test eltype(b) <: AbstractRGB && eltype(eltype(b)) == N0f8 && b[1] == RGB(1,0,0)
        img = fill(RGBA{Float32}(1,0,0,0.5), 1, 1)
        open(fn, "w") do file
            show(file, MIME("image/png"), img, minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test isa(b, Matrix{RGBA{N0f8}}) && b[1] == RGBA{N0f8}(1,0,0,0.5)
        img = Gray.([0.1 0.2; -0.5 0.8])
        open(fn, "w") do file
            show(file, MIME("image/png"), img, minpixels=0, maxpixels=typemax(Int))
        end
        b = load(fn)
        @test isa(b, Matrix{Gray{N0f8}}) && b == Gray{N0f8}[0.1 0.2; 0 0.8]
    end
    @testset "small images (expansion)" begin
        A = N0f8[0.01 0.99; 0.25 0.75]
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            show(file, MIME("image/png"), Gray.(A), minpixels=5, maxpixels=typemax(Int))
        end
        @test load(fn) == A[[1,1,2,2],[1,1,2,2]]
    end
end

# We do this after the FileIO backend has gotten comfortable.
const have_restrict = Ref{Bool}(false)
const restrict_mod = Ref{Module}()
for mod in values(Base.loaded_modules)
    if string(mod) == "ImageTransformations"
        have_restrict[] = true
        restrict_mod[] = mod
    end
end

if have_restrict[]
    @info "Tests will use restrict from ImageTransformations"
    const restrict = restrict_mod[].restrict
end

@testset "Big images and matrices" begin
    @testset "big images (use of restrict)" begin
        A = N0f8[0.01 0.4 0.99; 0.25 0.8 0.75; 0.6 0.2 0.0]
        if have_restrict[]
            Ar = restrict(A)
        else
            Ar = N0f8[0.01 0.99; 0.6 0.0]
        end
        fn = joinpath(workdir, "writemime.png")
        open(fn, "w") do file
            if !have_restrict[]
                @test_logs (:info, r"^For better quality") show(file, MIME("image/png"), Gray.(A), minpixels=0, maxpixels=5)
            else
                show(file, MIME("image/png"), Gray.(A), minpixels=0, maxpixels=5)
            end
        end
        @test load(fn) == N0f8.(Ar)
        # a genuinely big image (tests the defaults)
        abig = colorview(Gray, normedview(rand(UInt8, 1024, 1023)))
        fn = joinpath(workdir, "big.png")
        open(fn, "w") do file
            show(file, MIME("image/png"), abig, maxpixels=10^6)
        end
        b = load(fn)
        if have_restrict[]
            btmp = restrict(abig, (1,2))
        else
            btmp = abig[1:2:end,1:2:end]
        end
        @test b == N0f8.(btmp)
    end
    @testset "display matrix of images" begin
        img() = colorview(Gray, rand([0.250 0.5; 0.75 1.0], rand(2:10), rand(2:10)))
        io = IOBuffer()
        # test that these methods don't fail
        show(io, MIME"text/html"(), [img() for i=1:2])
        show(io, MIME"text/html"(), [img() for i=1:2, j=1:2])
        show(io, MIME"text/html"(), [img() for i=1:2, j=1:2, k=1:2])
    end
    @testset "display matrix of 1-D images" begin
        flat_imgs = [zeros(Gray{Float32}, 1)]
        io = IOBuffer()
        # These methods should not invoke the ImageShow.jl display code, but they
        # used to throw errors: https://github.com/JuliaImages/Images.jl/issues/623
        @test !applicable(ImageShow._show_odd, io, MIME"text/html"(), flat_imgs)
        @test !applicable(ImageShow._show_even, io, MIME"text/html"(), flat_imgs)
    end
    @testset "Non-1 indexing" begin
        A = N0f8[0.01 0.99; 0.25 0.75]
        Aoff = OffsetArray(A, 0:1, 2:3)
        fn = joinpath(workdir, "oas.png")
        open(fn, "w") do file
            show(file, MIME("image/png"), Gray.(Aoff), minpixels=5, maxpixels=typemax(Int))
        end
        @test load(fn) == A[[1,1,2,2],[1,1,2,2]]
        # Also test a type that doesn't have a specialization so as to trigger the generic
        # fallback
        A = PaddedView(0, reshape([1], 1, 1), (0:1, 1:2))
        Ac = collect(A)
        open(fn, "w") do file
            show(file, MIME("image/png"), Gray.(A), minpixels=5, maxpixels=typemax(Int))
        end
        @test load(fn) == Ac[[1,1,2,2],[1,1,2,2]]
    end
end
try
    # if this fails, it is not our fault
    rm(workdir, recursive=true)
catch
    # do nothing
end

nothing


text:
#-------------------------------------------------------------------
#* EMSO Model Library (EML) Copyright (C) 2004 - 2007 ALSOC.
#*
#* This LIBRARY is free software; you can distribute it and/or modify
#* it under the therms of the ALSOC FREE LICENSE as available at
#* http://www.enq.ufrgs.br/alsoc.
#*
#* EMSO Copyright (C) 2004 - 2007 ALSOC, original code
#* from http://www.rps.eng.br Copyright (C) 2002-2004.
#* All rights reserved.
#*
#* EMSO is distributed under the therms of the ALSOC LICENSE as
#* available at http://www.enq.ufrgs.br/alsoc.
#*
#*----------------------------------------------------------------------
#* File containg models of columns: distillation, stripping, absorbers
#* rectifier, ....
#*
#* The default nomenclature is:
#*		Type_Column_reboilertype_condensertyper
#*
#* where:
#*	Type = refluxed or reboiled or section
#*	Column = Stripping, Absorption, Rectifier, Distillation
#*	Reboiler type (if exists) = kettle or thermosyphon
#*	Condenser type (if exists) = with subccoling or without subcooling
#*
#*-----------------------------------------------------------------------
#* Author: Paula B. Staudt
#* $Id: column.mso 511 2008-05-12 17:25:33Z paula $
#*---------------------------------------------------------------------
type Section_Column3
	Section_Column3()=begin
		new(
			SectionColumnBasic(),
			stream(Dict{Symbol,Any}(
				:Brief=>"Feed stream",
				:PosX=>0,
				:PosY=>0.45
			)),
			stream(Dict{Symbol,Any}(
				:Brief=>"Feed stream",
				:PosX=>0,
				:PosY=>0.65
			)),
			stream(Dict{Symbol,Any}(
				:Brief=>"Feed stream",
				:PosX=>0,
				:PosY=>0.85
			)),
			[
				:(FeedTrayOne.F= TRAYS([1:_base_1.NumberOfTrays]).Inlet.F),
				:(FeedTrayOne.T = TRAYS([1:_base_1.NumberOfTrays]).Inlet.T),
				:(FeedTrayOne.P = TRAYS([1:_base_1.NumberOfTrays]).Inlet.P),
				:(FeedTrayOne.z = TRAYS([1:_base_1.NumberOfTrays]).Inlet.z),
				:(FeedTrayOne.v = TRAYS([1:_base_1.NumberOfTrays]).Inlet.v),
				:(FeedTrayOne.h = TRAYS([1:_base_1.NumberOfTrays]).Inlet.h),
				:(FeedTrayTwo.F= TRAYS([1:_base_1.NumberOfTrays]).Inlet.F),
				:(FeedTrayTwo.T = TRAYS([1:_base_1.NumberOfTrays]).Inlet.T),
				:(FeedTrayTwo.P = TRAYS([1:_base_1.NumberOfTrays]).Inlet.P),
				:(FeedTrayTwo.z = TRAYS([1:_base_1.NumberOfTrays]).Inlet.z),
				:(FeedTrayTwo.v = TRAYS([1:_base_1.NumberOfTrays]).Inlet.v),
				:(FeedTrayTwo.h = TRAYS([1:_base_1.NumberOfTrays]).Inlet.h),
				:(FeedTrayThree.F= TRAYS([1:_base_1.NumberOfTrays]).Inlet.F),
				:(FeedTrayThree.T = TRAYS([1:_base_1.NumberOfTrays]).Inlet.T),
				:(FeedTrayThree.P = TRAYS([1:_base_1.NumberOfTrays]).Inlet.P),
				:(FeedTrayThree.z = TRAYS([1:_base_1.NumberOfTrays]).Inlet.z),
				:(FeedTrayThree.v = TRAYS([1:_base_1.NumberOfTrays]).Inlet.v),
				:(FeedTrayThree.h = TRAYS([1:_base_1.NumberOfTrays]).Inlet.h),
				:(0*"mol/h"= TRAYS([1:_base_1.NumberOfTrays]).Inlet.F),
				:(300*"K" = TRAYS([1:_base_1.NumberOfTrays]).Inlet.T),
				:(1*"atm" = TRAYS([1:_base_1.NumberOfTrays]).Inlet.P),
				:(0.1 = TRAYS([1:_base_1.NumberOfTrays]).Inlet.z),
				:(0 = TRAYS([1:_base_1.NumberOfTrays]).Inlet.v),
				:(0*"J/mol" = TRAYS([1:_base_1.NumberOfTrays]).Inlet.h),
			],
			[
				"FeedTrayTop Inlet Flow","FeedTrayTop Inlet Temperature","FeedTrayTop Inlet Pressure","FeedTrayTop Inlet Composition","FeedTrayTop Inlet Vapour Fraction","FeedTrayTop Inlet Enthalpy","FeedTrayBottom Inlet Flow","FeedTrayBottom Inlet Temperature","FeedTrayBottom Inlet Pressure","FeedTrayBottom Inlet Composition","FeedTrayBottom Inlet Vapour Fraction","FeedTrayBottom Inlet Enthalpy","FeedTrayBottom Inlet Flow","FeedTrayBottom Inlet Temperature","FeedTrayBottom Inlet Pressure","FeedTrayBottom Inlet Composition","FeedTrayBottom Inlet Vapour Fraction","FeedTrayBottom Inlet Enthalpy","Inlet Tray - Flow Sealed","Inlet Tray -  Temperature","Inlet Tray -  Pressure","Inlet Tray -  Composition","Inlet Tray -  Vapour Fraction","Inlet Tray -  Enthalpy",
			],
			[:FeedTrayOne,:FeedTrayTwo,:FeedTrayThree,]
		)
	end
	_base_1::SectionColumnBasic
	FeedTrayOne::stream
	FeedTrayTwo::stream
	FeedTrayThree::stream
	equations::Array{Expr,1}
	equationNames::Array{String,1}
	variables::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export Section_Column3
function set(in::Section_Column3)
	_base_1.NumberOfFeeds = 3

end
function setEquationFlow(in::Section_Column3)
	if i == FeedTrayLocation(1)
		addEquation(1)
		addEquation(2)
		addEquation(3)
		addEquation(4)
		addEquation(5)
		addEquation(6)
	else
		if i == FeedTrayLocation(2)
			addEquation(7)
			addEquation(8)
			addEquation(9)
			addEquation(10)
			addEquation(11)
			addEquation(12)
		else
			if i == FeedTrayLocation(3)
				addEquation(13)
				addEquation(14)
				addEquation(15)
				addEquation(16)
				addEquation(17)
				addEquation(18)
			else
				addEquation(19)
				addEquation(20)
				addEquation(21)
				addEquation(22)
				addEquation(23)
				addEquation(24)
			end
		end
	end
end
function atributes(in::Section_Column3,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=true
	fields[:Icon]="icon/SectionColumn"
	fields[:Brief]="Model of a column section."
	fields[:Info]="== Model of a column section containing ==
* NumberOfTrays TRAYS.
* Three Feed Inlets.

== Specify ==
* the feed stream of each tray (Inlet);
* the Murphree eficiency for each tray Emv;
* the InletLiquid stream of the top tray;
* the InletVapour stream of the bottom tray.

== Initial Conditions ==
* the TRAYS temperature (OutletLiquid.T);
* the TRAYS liquid level (Level) OR the TRAYS liquid flow (OutletLiquid.F);
* (NoComps - 1) OutletLiquid (OR OutletVapour) compositions for each tray.
"
	drive!(fields,_)
	return fields
end
Section_Column3(_::Dict{Symbol,Any})=begin
	newModel=Section_Column3()
	newModel.attributes=atributes(newModel,_)
	newModel
end

ext: jl


text:
##############################################################################
##
## PooledDataArray type definition
##
## An AbstractDataArray with efficient storage when values are repeated. A
## PDA wraps an array of unsigned integers, which are used to index into a
## compressed pool of values. NA's are 0's in the refs array.  The unsigned
## integer type used for the refs array defaults to Uint32.  The `compact`
## function converts to a smallest integer size that will index the entire pool.
##
## TODO: Allow ordering of factor levels
## TODO: Add metadata for dummy conversion
##
##############################################################################

# This is used as a wrapper during PooledDataArray construction only, to distinguish
# arrays of pool indices from normal arrays
type RefArray{R<:Integer,N}
    a::Array{R,N}
end

type PooledDataArray{T, R<:Integer, N} <: AbstractDataArray{T, N}
    refs::Array{R, N}
    pool::Vector{T}

    function PooledDataArray(rs::RefArray{R, N},
                             p::Vector{T})
        # refs mustn't overflow pool
        if length(rs.a) > 0 && maximum(rs.a) > prod(size(p))
            error("Reference array points beyond the end of the pool")
        end
        new(rs.a,p)
    end
end
typealias PooledDataVector{T,R} PooledDataArray{T,R,1}
typealias PooledDataMatrix{T,R} PooledDataArray{T,R,2}

##############################################################################
##
## PooledDataArray constructors
##
# How do you construct a PooledDataArray from an Array?
# From the same sigs as a DataArray!
#
# Algorithm:
# * Start with:
#   * A null pool
#   * A pre-allocated refs
#   * A hash from T to Int
# * Iterate over d
#   * If value of d in pool already, set the refs accordingly
#   * If value is new, add it to the pool, then set refs
##############################################################################

# Echo inner constructor as an outer constructor
function PooledDataArray{T,R<:Integer,N}(refs::RefArray{R, N},
                                         pool::Vector{T})
    PooledDataArray{T,R,N}(refs, pool)
end

# A no-op constructor
PooledDataArray(d::PooledDataArray) = d

# Constructor from array, w/ pool, missingness, and ref type
function PooledDataArray{T,R<:Integer,N}(d::AbstractArray{T, N},
                                         pool::Vector{T},
                                         m::AbstractArray{Bool, N},
                                         r::Type{R} = DEFAULT_POOLED_REF_TYPE)
    if length(pool) > typemax(R)
        error("Cannot construct a PooledDataVector with type $R with a pool of size $(length(pool))")
    end

    newrefs = Array(R, size(d))
    poolref = Dict{T, R}()

    # loop through once to fill the poolref dict
    for i = 1:length(pool)
        poolref[pool[i]] = i
    end

    # fill in newrefs
    for i = 1:length(d)
        if m[i]
            newrefs[i] = 0
        else
            newrefs[i] = get(poolref, d[i], 0)
        end
    end
    return PooledDataArray(RefArray(newrefs), pool)
end

# Constructor from array, w/ missingness and ref type
function PooledDataArray{T,R<:Integer,N}(d::AbstractArray{T, N},
                                         m::AbstractArray{Bool, N},
                                         r::Type{R} = DEFAULT_POOLED_REF_TYPE)
    pool = sort(unique(d[!m]))
    PooledDataArray(d, pool, m, r)
end

# Construct an all-NA PooledDataVector of a specific type
PooledDataArray(t::Type, dims::Int...) = PooledDataArray(Array(t, dims...), trues(dims...))
PooledDataArray{R<:Integer}(t::Type, r::Type{R}, dims::Int...) = PooledDataArray(Array(t, dims...), trues(dims...), r)

# Construct an empty PooledDataVector of a specific type
PooledDataArray(t::Type) = PooledDataArray(similar(Array(t,1),0), trues(0))
PooledDataArray{R<:Integer}(t::Type, r::Type{R}) = PooledDataArray(similar(Array(t,1),0), trues(0), r)

# Convert a BitArray to an Array{Bool} (m = missingness)
# For some reason an additional method is needed but even that doesn't work
# For a BitArray a refs type of Uint8 will always be sufficient as the size of the pool is 0, 1 or 2
PooledDataArray{N}(d::BitArray{N}) = PooledDataArray(bitunpack(d), falses(size(d)), Uint8)
PooledDataArray{N}(d::BitArray{N}, m::AbstractArray{Bool, N}) = PooledDataArray(bitunpack(d), m, Uint8)

# Convert a DataArray to a PooledDataArray
PooledDataArray{T,R<:Integer}(da::DataArray{T},
                              r::Type{R} = DEFAULT_POOLED_REF_TYPE) = PooledDataArray(da.data, da.na, r)
PooledDataArray{T,R<:Integer}(da::DataArray{T},
                              pool::Vector{T},
                              r::Type{R} = DEFAULT_POOLED_REF_TYPE) = PooledDataArray(da.data, pool, da.na, r)

# Convert a Array{T} to a PooledDataArray
PooledDataArray{T,R<:Integer}(d::Array{T},
                              r::Type{R} = DEFAULT_POOLED_REF_TYPE) = PooledDataArray(d, falses(size(d)), r)
PooledDataArray{T,R<:Integer}(d::Array{T},
                              pool::Vector{T},
                              r::Type{R} = DEFAULT_POOLED_REF_TYPE) = PooledDataArray(d, pool, falses(size(d)), r)

# Explicitly convert Ranges into a PooledDataVector
PooledDataArray{R<:Integer}(rs::Ranges,
                            r::Type{R} = DEFAULT_POOLED_REF_TYPE) = PooledDataArray([rs], falses(length(rs)), r)

# Initialized constructors with 0's, 1's
for (f, basef) in ((:pdatazeros, :zeros), (:pdataones, :ones))
    @eval begin
        ($f)(dims::Int...) = PooledDataArray(($basef)(dims...), falses(dims...))
        ($f)(t::Type, dims::Int...) = PooledDataArray(($basef)(t, dims...), falses(dims...))
        ($f){R<:Integer}(t::Type, r::Type{R}, dims::Int...) = PooledDataArray(($basef)(t, dims...), falses(dims...), r)
    end
end

# Initialized constructors with false's or true's
for (f, basef) in ((:pdatafalses, :falses), (:pdatatrues, :trues))
    @eval begin
        ($f)(dims::Int...) = PooledDataArray(($basef)(dims...), falses(dims...), Uint8)
        ($f){R<:Integer}(r::Type{R}, dims::Int...) = PooledDataArray(($basef)(dims...), falses(dims...), r)
    end
end

# Super hacked-out constructor: PooledDataVector[1, 2, 2, NA]
function getindex(::Type{PooledDataVector}, vals...)
    # For now, just create a DataVector and then convert it
    # TODO: Rewrite for speed
    PooledDataArray(DataVector[vals...])
end

##############################################################################
##
## Basic size properties of all Data* objects
##
##############################################################################

size(pda::PooledDataArray) = size(pda.refs)
length(pda::PooledDataArray) = length(pda.refs)
endof(pda::PooledDataArray) = endof(pda.refs)

##############################################################################
##
## Copying Data* objects
##
##############################################################################

copy(pda::PooledDataArray) = PooledDataArray(RefArray(copy(pda.refs)),
                                             copy(pda.pool))
# TODO: Implement copy_to()

##############################################################################
##
## Predicates, including the new isna()
##
##############################################################################

function isnan(pda::PooledDataArray)
    PooledDataArray(RefArray(copy(pda.refs)), isnan(pda.pool))
end

function isfinite(pda::PooledDataArray)
    PooledDataArray(RefArray(copy(pda.refs)), isfinite(pda.pool))
end

isna(pda::PooledDataArray) = pda.refs .== 0

##############################################################################
##
## PooledDataArray utilities
##
## TODO: Add methods with these names for DataArray's
##       Decide whether levels() or unique() is primitive. Make the other
##       an alias.
##  Tom: I don't think levels and unique are the same. R doesn't include NA's
##       with levels, but it does with unique. Having these different is
##       useful.
##
##############################################################################

function compact{T,R<:Integer,N}(d::PooledDataArray{T,R,N})
    sz = length(d.pool)

    REFTYPE = sz <= typemax(Uint8)  ? Uint8 :
              sz <= typemax(Uint16) ? Uint16 :
              sz <= typemax(Uint32) ? Uint32 :
                                      Uint64

    if REFTYPE == R
        return d
    end

    newrefs = convert(Array{REFTYPE, N}, d.refs)
    PooledDataArray(RefArray(newrefs), d.pool)
end

# Convert a PooledDataVector{T} to a DataVector{T}
function values{T}(pda::PooledDataArray{T})
    res = DataArray(T, size(pda)...)
    for i in 1:length(pda)
        r = pda.refs[i]
        if r == 0
            res[i] = NA
        else
            res[i] = pda.pool[r]
        end
    end
    return res
end
DataArray(pda::PooledDataArray) = values(pda)
values(da::DataArray) = copy(da)
values(a::Array) = copy(a)

function unique{T}(x::PooledDataArray{T})
    if any(x.refs .== 0)
        n = length(x.pool)
        d = Array(T, n + 1)
        for i in 1:n
            d[i] = x.pool[i]
        end
        m = falses(n + 1)
        m[n + 1] = true
        return DataArray(d, m)
    else
        return DataArray(copy(x.pool), falses(length(x.pool)))
    end
end
levels{T}(pda::PooledDataArray{T}) = pda.pool

function unique{T}(adv::AbstractDataVector{T})
  values = Dict{Union(T, NAtype), Bool}()
  for i in 1:length(adv)
    values[adv[i]] = true
  end
  unique_values = collect(keys(values))
  res = DataArray(T, length(unique_values))
  for i in 1:length(unique_values)
    res[i] = unique_values[i]
  end
  return res
end
levels{T}(adv::AbstractDataVector{T}) = unique(adv)

get_indices{T,R}(x::PooledDataArray{T,R}) = x.refs

function index_to_level{T,R}(x::PooledDataArray{T,R})
    d = Dict{R, T}()
    for i in one(R):convert(R, length(x.pool))
        d[i] = x.pool[i]
    end
    return d
end

function level_to_index{T,R}(x::PooledDataArray{T,R})
    d = Dict{T, R}()
    for i in one(R):convert(R, length(x.pool))
        d[x.pool[i]] = i
    end
    d
end

function PooledDataArray{S,R,N}(x::PooledDataArray{S,R,N},
                                newpool::Vector{S})
    # QUESTION: should we have a ! version of this? If so, needs renaming?
    tidx::Array{R} = findat(x.pool, newpool)
    refs = zeros(R, length(x))
    for i in 1:length(refs)
        if x.refs[i] != 0
            refs[i] = tidx[x.refs[i]]
        end
    end
    PooledDataArray(RefArray(refs), newpool)
end

myunique(x::AbstractVector) = x[sort(unique(findat(x, x)))]  # gets the ordering right
myunique(x::AbstractDataVector) = myunique(removeNA(x))   # gets the ordering right; removes NAs

function set_levels{T,R}(x::PooledDataArray{T,R}, newpool::AbstractVector)
    pool = myunique(newpool)
    refs = zeros(R, length(x))
    tidx::Array{R} = findat(newpool, pool)
    tidx[isna(newpool)] = 0
    for i in 1:length(refs)
        if x.refs[i] != 0
            refs[i] = tidx[x.refs[i]]
        end
    end
    return PooledDataArray(RefArray(refs), pool)
end

function set_levels!{T,R}(x::PooledDataArray{T,R}, newpool::AbstractVector{T})
    if newpool == myunique(newpool) # no NAs or duplicates
        x.pool = newpool
        return x
    else
        x.pool = myunique(newpool)
        tidx::Array{R} = findat(newpool, x.pool)
        tidx[isna(newpool)] = 0
        for i in 1:length(x.refs)
            if x.refs[i] != 0
                x.refs[i] = tidx[x.refs[i]]
            end
        end
        return x
    end
end

function set_levels(x::PooledDataArray, d::Dict)
    newpool = copy(DataArray(x.pool))
    # An NA in `v` is put in the pool; that will cause it to become NA
    for (k,v) in d
        idx = findin(newpool, [k])
        if length(idx) == 1
            newpool[idx[1]] = v
        end
    end
    set_levels(x, newpool)
end

function set_levels!{T,R}(x::PooledDataArray{T,R}, d::Dict{T,T})
    for (k,v) in d
        idx = findin(x.pool, [k])
        if length(idx) == 1
            x.pool[idx[1]] = v
        end
    end
    x
end

function set_levels!{T,R}(x::PooledDataArray{T,R}, d::Dict{T,Any}) # this version handles NAs in d's values
    newpool = copy(DataArray(x.pool))
    # An NA in `v` is put in the pool; that will cause it to become NA
    for (k,v) in d
        idx = findin(newpool, [k])
        if length(idx) == 1
            newpool[idx[1]] = v
        end
    end
    set_levels!(x, newpool)
end

reorder(x::PooledDataArray) = PooledDataArray(x, sort(levels(x)))  # just re-sort the pool

reorder(x::PooledDataArray, y::AbstractVector...) = reorder(mean, x, y...)

reorder(fun::Function, x::PooledDataArray, y::AbstractVector...) =
    reorder(fun, x, DataFrame({y...}))

reverse(x::PooledDataArray) = PooledDataArray(RefArray(reverse(x.refs)), x.pool)

##############################################################################
##
## similar()
##
##############################################################################

similar{T,R}(pda::PooledDataArray{T,R}, S, dims::Dims) =
    PooledDataArray(RefArray(zeros(R, dims)), S[])

##############################################################################
##
## find()
##
##############################################################################

find(pdv::PooledDataVector{Bool}) = find(values(pdv))

##############################################################################
##
## getindex()
##
##############################################################################

# pda[SingleItemIndex]
function getindex(pda::PooledDataArray, i::Real)
    if pda.refs[i] == 0
        return NA
    else
        return pda.pool[pda.refs[i]]
    end
end

# pda[MultiItemIndex]
function getindex(pda::PooledDataArray, inds::AbstractDataVector{Bool})
    inds = find(replaceNA(inds, false))
    return PooledDataArray(RefArray(pda.refs[inds]), copy(pda.pool))
end
function getindex(pda::PooledDataArray, inds::AbstractDataVector)
    inds = removeNA(inds)
    return PooledDataArray(RefArray(pda.refs[inds]), copy(pda.pool))
end
function getindex(pda::PooledDataArray, inds::Union(Vector, BitVector, Ranges))
    return PooledDataArray(RefArray(pda.refs[inds]), copy(pda.pool))
end

# pdm[SingleItemIndex, SingleItemIndex)
function getindex(pda::PooledDataArray, i::Real, j::Real)
    if pda.refs[i, j] == 0
        return NA
    else
        return pda.pool[pda.refs[i, j]]
    end
end

# pda[SingleItemIndex, MultiItemIndex]
function getindex(pda::PooledDataArray, i::Real, col_inds::AbstractDataVector{Bool})
    getindex(pda, i, find(replaceNA(col_inds, false)))
end
function getindex(pda::PooledDataArray, i::Real, col_inds::AbstractDataVector)
    getindex(pda, i, removeNA(col_inds))
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             i::Real,
             col_inds::Union(Vector, BitVector, Ranges))
    error("not yet implemented")
    PooledDataArray(RefArray(pda.refs[i, col_inds]), pda.pool[i, col_inds])
end

# pda[MultiItemIndex, SingleItemIndex]
function getindex(pda::PooledDataArray, row_inds::AbstractDataVector{Bool}, j::Real)
    getindex(pda, find(replaceNA(row_inds, false)), j)
end
function getindex(pda::PooledDataArray, row_inds::AbstractVector, j::Real)
    getindex(pda, removeNA(row_inds), j)
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             row_inds::Union(Vector, BitVector, Ranges),
             j::Real)
    error("not yet implemented")
    PooledDataArray(RefArray(pda.refs[row_inds, j]), pda.pool[row_inds, j])
end

# pda[MultiItemIndex, MultiItemIndex]
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector{Bool},
             col_inds::AbstractDataVector{Bool})
    getindex(pda, find(replaceNA(row_inds, false)), find(replaceNA(col_inds, false)))
end
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector{Bool},
             col_inds::AbstractDataVector)
    getindex(pda, find(replaceNA(row_inds, false)), removeNA(col_inds))
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector{Bool},
             col_inds::Union(Vector, BitVector, Ranges))
    getindex(pda, find(replaceNA(row_inds, false)), col_inds)
end
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector,
             col_inds::AbstractDataVector{Bool})
    getindex(pda, removeNA(row_inds), find(replaceNA(col_inds, false)))
end
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector,
             col_inds::AbstractDataVector)
    getindex(pda, removeNA(row_inds), removeNA(col_inds))
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             row_inds::AbstractDataVector,
             col_inds::Union(Vector, BitVector, Ranges))
    getindex(pda, removeNA(row_inds), col_inds)
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             row_inds::Union(Vector, BitVector, Ranges),
             col_inds::AbstractDataVector{Bool})
    getindex(pda, row_inds, find(replaceNA(col_inds, false)))
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
             row_inds::Union(Vector, BitVector, Ranges),
             col_inds::AbstractDataVector)
    getindex(pda, row_inds, removeNA(col_inds))
end
# TODO: Make inds::AbstractVector
function getindex(pda::PooledDataArray,
                  row_inds::Union(Vector, BitVector, Ranges),
                  col_inds::Union(Vector, BitVector, Ranges))
    error("not yet implemented")
    PooledDataArray(RefArray(pda.refs[row_inds, col_inds]), pda.pool[row_inds, col_inds])
end

##############################################################################
##
## setindex!() definitions
##
##############################################################################

function getpoolidx{T,R<:Union(Uint8, Uint16, Int8, Int16)}(pda::PooledDataArray{T,R}, val::Any)
    val::T = convert(T,val)
    pool_idx = findfirst(pda.pool, val)
    if pool_idx <= 0
        push!(pda.pool, val)
        pool_idx = length(pda.pool)
        if pool_idx > typemax(R)
            error("You're using a PooledDataArray with ref type $R, which can only hold $(int(typemax(R))) values,\n",
                  "and you just tried to add the $(typemax(R)+1)th reference.  Please change the ref type\n",
                  "to a larger int type, or use the default ref type ($DEFAULT_POOLED_REF_TYPE).")
        end
    end
    return pool_idx
end

function getpoolidx{T,R}(pda::PooledDataArray{T,R}, val::Any)
    val::T = convert(T,val)
    pool_idx = findfirst(pda.pool, val)
    if pool_idx <= 0
        push!(pda.pool, val)
        pool_idx = length(pda.pool)
    end
    return pool_idx
end

# x[SingleIndex] = NA
# TODO: Delete values from pool that no longer exist?
# Not a good idea.  Add another function called drop_unused_levels to do this.
# R has the convention that if f is a factor then factor(f) drops unused levels
function setindex!(x::PooledDataArray, val::NAtype, ind::Real)
    x.refs[ind] = 0
    return NA
end

# x[SingleIndex] = Single Item
# TODO: Delete values from pool that no longer exist?
function setindex!{T,R}(x::PooledDataArray{T,R}, val::Any, ind::Real)
    val = convert(T, val)
    x.refs[ind] = getpoolidx(x, val)
    return val
end

# x[MultiIndex] = NA
# TODO: Find a way to delete the next four methods
function setindex!(x::PooledDataArray{NAtype},
                val::NAtype,
                inds::AbstractVector{Bool})
    error("Don't use PooledDataVector{NAtype}'s")
end
function setindex!(x::PooledDataArray{NAtype},
                val::NAtype,
                inds::AbstractVector)
    error("Don't use PooledDataVector{NAtype}'s")
end
function setindex!(x::PooledDataArray, val::NAtype, inds::AbstractVector{Bool})
    inds = find(inds)
    x.refs[inds] = 0
    return NA
end
function setindex!(x::PooledDataArray, val::NAtype, inds::AbstractVector)
    x.refs[inds] = 0
    return NA
end

# pda[MultiIndex] = Multiple Values
function setindex!(pda::PooledDataArray,
                   vals::AbstractVector,
                   inds::AbstractVector{Bool})
    setindex!(pda, vals, find(inds))
end
function setindex!(pda::PooledDataArray,
                   vals::AbstractVector,
                   inds::AbstractVector)
    for (val, ind) in zip(vals, inds)
        pda[ind] = val
    end
    return vals
end

# pda[SingleItemIndex, SingleItemIndex] = NA
function setindex!{T,R}(pda::PooledDataMatrix{T,R}, val::NAtype, i::Real, j::Real)
    pda.refs[i, j] = zero(R)
    return NA
end
# pda[SingleItemIndex, SingleItemIndex] = Single Item
function setindex!{T,R}(pda::PooledDataMatrix{T,R}, val::Any, i::Real, j::Real)
    val = convert(T, val)
    pda.refs[i, j] = getpoolidx(pda, val)
    return val
end

##############################################################################
##
## show() and similar methods
##
##############################################################################

function string(x::PooledDataVector)
    tmp = join(x, ", ")
    return "[$tmp]"
end

# # Need setindex!()'s to make this work
# This is broken now because the inner show returns to the outer show.
# function show(io::IO, pda::PooledDataArray)
#     invoke(show, (Any, AbstractArray), io, pda)
#     print(io, "\nlevels: ")
#     print(io, levels(pda))
# end

##############################################################################
##
## Replacement operations
##
##############################################################################

function replace!(x::PooledDataArray{NAtype}, fromval::NAtype, toval::NAtype)
    NA # no-op to deal with warning
end
function replace!(x::PooledDataArray, fromval::NAtype, toval::NAtype)
    NA # no-op to deal with warning
end
function replace!{S, T}(x::PooledDataArray{S}, fromval::T, toval::NAtype)
    fromidx = findfirst(x.pool, fromval)
    if fromidx == 0
        error("can't replace a value not in the pool in a PooledDataVector!")
    end

    x.refs[x.refs .== fromidx] = 0

    return NA
end
function replace!{S, T}(x::PooledDataArray{S}, fromval::NAtype, toval::T)
    toidx = findfirst(x.pool, toval)
    # if toval is in the pool, just do the assignment
    if toidx != 0
        x.refs[x.refs .== 0] = toidx
    else
        # otherwise, toval is new, add it to the pool
        push!(x.pool, toval)
        x.refs[x.refs .== 0] = length(x.pool)
    end

    return toval
end
function replace!{R, S, T}(x::PooledDataArray{R}, fromval::S, toval::T)
    # throw error if fromval isn't in the pool
    fromidx = findfirst(x.pool, fromval)
    if fromidx == 0
        error("can't replace a value not in the pool in a PooledDataArray!")
    end

    # if toval is in the pool too, use that and remove fromval from the pool
    toidx = findfirst(x.pool, toval)
    if toidx != 0
        x.refs[x.refs .== fromidx] = toidx
        #x.pool[fromidx] = None    TODO: what to do here??
    else
        # otherwise, toval is new, swap it in
        x.pool[fromidx] = toval
    end

    return toval
end

##############################################################################
##
## Sorting can use the pool to speed things up
##
##############################################################################

# TODO handle sortperm for non-sorted keys
sortperm(pda::PooledDataArray) = groupsort_indexer(pda)[1]
function sortperm(pda::PooledDataArray)
    if issorted(pda.pool)
        return groupsort_indexer(pda)[1]
    else
        return sortperm(reorder!(copy(pda)))
    end
end

sortperm(pda::PooledDataArray, ::Sort.ReverseOrdering) = reverse(sortperm(pda))
sort(pda::PooledDataArray) = pda[sortperm(pda)]
sort(pda::PooledDataArray, ::Sort.ReverseOrdering) = pda[reverse(sortperm(pda))]
type FastPerm{O<:Sort.Ordering,V<:AbstractVector} <: Sort.Ordering
    ord::O
    vec::V
end
FastPerm{O<:Sort.Ordering,V<:AbstractVector}(o::O,v::V) = FastPerm{O,V}(o,v)
sortperm{V}(x::AbstractVector, a::Sort.Algorithm, o::FastPerm{Sort.ForwardOrdering,V}) = x[sortperm(o.vec)]
sortperm{V}(x::AbstractVector, a::Sort.Algorithm, o::FastPerm{Sort.ReverseOrdering,V}) = x[reverse(sortperm(o.vec))]
Perm{O<:Sort.Ordering}(o::O, v::PooledDataVector) = FastPerm(o, v)

##############################################################################
##
## PooledDataVecs: EXPLANATION SHOULD GO HERE
##
##############################################################################

function PooledDataVecs{S,Q<:Integer,R<:Integer,N}(v1::PooledDataArray{S,Q,N},
                                                   v2::PooledDataArray{S,R,N})
    pool = sort(unique([v1.pool, v2.pool]))
    sz = length(pool)

    REFTYPE = sz <= typemax(Uint8)  ? Uint8 :
              sz <= typemax(Uint16) ? Uint16 :
              sz <= typemax(Uint32) ? Uint32 :
                                      Uint64

    tidx1 = convert(Vector{REFTYPE}, findat(v1.pool, pool))
    tidx2 = convert(Vector{REFTYPE}, findat(v2.pool, pool))
    refs1 = zeros(REFTYPE, length(v1))
    refs2 = zeros(REFTYPE, length(v2))
    for i in 1:length(refs1)
        if v1.refs[i] != 0
            refs1[i] = tidx1[v1.refs[i]]
        end
    end
    for i in 1:length(refs2)
        if v2.refs[i] != 0
            refs2[i] = tidx2[v2.refs[i]]
        end
    end
    return (PooledDataArray(RefArray(refs1), pool),
            PooledDataArray(RefArray(refs2), pool))
end

function PooledDataVecs{S,R<:Integer,N}(v1::PooledDataArray{S,R,N},
                                        v2::AbstractArray{S,N})
    return PooledDataVecs(v1,
                          PooledDataArray(v2))
end
function PooledDataVecs{S,R<:Integer,N}(v1::PooledDataArray{S,R,N},
                                        v2::AbstractArray{S,N})
    return PooledDataVecs(v1,
                          PooledDataArray(v2))
end

####
function PooledDataVecs{S,R<:Integer,N}(v1::AbstractArray{S,N},
                                        v2::PooledDataArray{S,R,N})
    return PooledDataVecs(PooledDataArray(v1),
                          v2)
end

function PooledDataVecs(v1::AbstractArray,
                        v2::AbstractArray)

    ## Return two PooledDataVecs that share the same pool.

    ## TODO: allow specification of REFTYPE
    refs1 = Array(DEFAULT_POOLED_REF_TYPE, size(v1))
    refs2 = Array(DEFAULT_POOLED_REF_TYPE, size(v2))
    poolref = Dict{promote_type(eltype(v1), eltype(v2)), DEFAULT_POOLED_REF_TYPE}()
    maxref = 0

    # loop through once to fill the poolref dict
    for i = 1:length(v1)
        if !isna(v1[i])
            poolref[v1[i]] = 0
        end
    end
    for i = 1:length(v2)
        if !isna(v2[i])
            poolref[v2[i]] = 0
        end
    end

    # fill positions in poolref
    pool = sort(collect(keys(poolref)))
    i = 1
    for p in pool
        poolref[p] = i
        i += 1
    end

    # fill in newrefs
    zeroval = zero(DEFAULT_POOLED_REF_TYPE)
    for i = 1:length(v1)
        if isna(v1[i])
            refs1[i] = zeroval
        else
            refs1[i] = poolref[v1[i]]
        end
    end
    for i = 1:length(v2)
        if isna(v2[i])
            refs2[i] = zeroval
        else
            refs2[i] = poolref[v2[i]]
        end
    end

    return (PooledDataArray(RefArray(refs1), pool),
            PooledDataArray(RefArray(refs2), pool))
end


text:
### A Pluto.jl notebook ###
# v0.17.1

using Markdown
using InteractiveUtils

# ╔═╡ 4cb4aa50-3e01-11eb-3460-5f109773492b
#export
using Base.Docs

# ╔═╡ 6c0aeda1-81d7-469d-8443-bbf347feede6
#export
using TOML

# ╔═╡ 151ec8b0-2b27-11eb-1ec2-a7c5e4c13db9
#export
using Markdown

# ╔═╡ 07b9d565-4690-4e48-bcc2-05428b665ca1
#export
using Distributed

# ╔═╡ 77a9e510-4ae2-44a2-9536-17a89a54d6f6
#export
using ProgressMeter

# ╔═╡ 295d813f-1e85-4b2e-a953-1bd0087a85e2
#hide
using PlutoTest

# ╔═╡ 9930ad11-84da-45e0-95dc-f46c4d48174a
#hide
using PlutoUI

# ╔═╡ b068dfd2-0eb3-11eb-109a-d1b6ef1eeca0
#export
include("../src/Export.jl")

# ╔═╡ 27ff1d70-1201-11eb-2003-27cb52571be6
#export
include("../src/CodeRunner.jl")

# ╔═╡ 2bc37e16-7ff5-430b-b29c-571db5e9b6b5
#export
include("../src/Common.jl")

# ╔═╡ 1d83078e-2024-11eb-0e5f-51310d134662
#export
import Pluto: Notebook, Cell, load_notebook_nobackup

# ╔═╡ fcfaccaf-cc88-44bb-8971-c1328d8599ee
#hide
TableOfContents()

# ╔═╡ 25ff264e-3ec5-11eb-362c-07b4e24c635a
md"## Lower Level Entities(Structs, methods etc.)
These are the objects on which nbdev's Documenter module was built. You can use it to extend nbdev but these are automtically used by Nbdev's internal engine to generate code files for you."

# ╔═╡ 28951cc5-0f45-48fc-965a-4257de0332fa
begin
f = joinpath("..", "Project.toml")
path = "https://github.com/sapal6/Nbdev.jl"
res = Dict("github_url" => "https://github.com/sapal6/Nbdev.jl")
end

# ╔═╡ c371d5ec-8d89-4f33-8b6d-9add8e15938f
#noop
@test Common.getsetting(f, "settings") == res

# ╔═╡ c099d54a-38df-46ee-8fb1-37c2b90c1df2
#noop
@test Common.getsettings(f, ["settings", "github_url"]) == res["github_url"]

# ╔═╡ bf4e47f0-3ec5-11eb-1b65-5fe2e7a88ff1
md"## Section type"

# ╔═╡ 5a2e9790-201f-11eb-0df4-f90b3cc54f20
#export
begin
"""
> struct Section--> This is like the section of a page and is made up of one or multiple lines.
> * Fields:
>   * line--> String which makes up a section
"""
Base.@kwdef mutable struct Section
	line::String=""
end

"""
> Section(line)--> Helper function to call the constructor of the type Section
"""
Section(line)=Section(line=line)
"""
> line(section::Section)--> Getter for accessing the underlying field line.
"""
line(section::Section)=section.line
end

# ╔═╡ f1318ca0-3f97-11eb-3727-3f71064c77bf
md"## Example"

# ╔═╡ cfb6d710-3f97-11eb-31ed-6daa12cf592e
section=Section("This is a test line")

# ╔═╡ 610d9263-645a-46aa-8f16-12e90787b7bc
#noop
@test line(section) == "This is a test line"

# ╔═╡ 309585e2-3f93-11eb-0873-d9b0d7a6200e
md"## Page Type"

# ╔═╡ 28a4e100-17ac-11eb-172c-2d0e73460caa
#export
begin
"""
> Page--> The type that represents a page in a document.
> * Fields:
>   * sections--> Array of Section type.
>   * name--> Name of the module being documented.
"""
struct Page
	sections::Array{Section,1}
    name::AbstractString
end

"""
> Page--> Helper function to create constructer for Page type.
"""
Page(sections, path)=Page(sections=sections, path=path)

"""
> sections--> Getter for accessing the underlying field sections of Page.
"""
sections(p::Page)=p.sections

"""
> name--> Getter for accessing the underlying field name of Page.
"""
name(p::Page)=p.name
end

# ╔═╡ 997a4950-3f94-11eb-2e66-d9a70cc175d1
md"A 📃 is made up of several sections. These sections can be example codes, text ,graphs(to be implemented) and function documentation and several pages. The Page type helps in implementing this concept into code."

# ╔═╡ ea16dab0-3f97-11eb-19b5-0f4998ba3e39
md"#### Example"

# ╔═╡ 11057000-3f98-11eb-27aa-ffb097ca25ed
begin
page=Page([Section("iny, meeny, miny mo"), Section("🙈 🙊 🙉")], "Monkey module")
sections(page)
end

# ╔═╡ 7e959872-421d-11eb-3784-2f110840a628
page

# ╔═╡ dd721172-4443-11eb-3833-3f74ff47dd42
typeof(page)

# ╔═╡ 8d851420-08af-11eb-26b0-63501c72011a
#hide
#cd("../nbs")

# ╔═╡ 9478dd20-08af-11eb-1663-a7db573d2187
#hide
pwd()

# ╔═╡ 74879c7e-0eeb-11eb-31bf-251621d154a6
#hide
function test(x)
	x+1
end

# ╔═╡ e2952860-1ade-11eb-20ca-091a45fab2f2
#hide
test(1)

# ╔═╡ d9fffed0-2f3f-11eb-16b4-4b2778f792d9
#hide
import Pluto

# ╔═╡ 2fc24ff0-11f7-11eb-18a8-1b2b989fa189
#hide
#testNb=Pluto.load_notebook("02_documenter.jl")

# ╔═╡ 743238d0-1918-11eb-3dfc-6f30db92923c
md"## run_and_update "

# ╔═╡ 8691e572-1918-11eb-011c-639d3617e076
#export
"""
> runandupdatenb(fn::AbstractString)--> Run the notebook in the supplied path and update the notebook with the output of each cell.
"""
function runandupdatenb(fn::AbstractString)
	nb=load_notebook_nobackup(fn)
	return CodeRunner.executecode(nb)
end

# ╔═╡ 5001a5b0-11ff-11eb-054a-6921da78afa3
md"`runandupdatenb` uses the `load_notebook_nobackup` function in Pluto.jl. There are multiple ways to achieve what this function achieves without depending on Pluto internals but for now this was the quickiest way to achieve this.

**Note-**Depending on Pluto internal to run and update a notebook makes it difficult to run unit test for this function from within pluto itself because a Pluto cell is treated as worker 2, 3 and so on but when you do load_notebook() then it spins up another worker. Only worker 1 can spwan further processes and not a notebook cell."

# ╔═╡ f1d7ed22-1f8a-11eb-035d-6de2cb48bb8b
#hide
Markdown.html(md"```func test end```")

# ╔═╡ 6b9cb0f0-1f5e-11eb-1e15-9f0c8295b59f
#hide
#code=testNb.cells[13].code

# ╔═╡ c3d2cf20-1f8b-11eb-0381-01270b1494b3
#hide
#cleanedCode=Export.strip(Export.strip(code,"\n"), "\n")

# ╔═╡ 1d71bb00-1f8b-11eb-31de-69aebd625201
#hide
#string("
",cleanedCode,"
\n")

# ╔═╡ bda9c5a0-1f8a-11eb-396b-97f97add91db
md"## stitchcode"

# ╔═╡ 00989200-25d6-11eb-3139-8dd2ca0346f8
#hide
#TODO:not used as of now
grabFuncSig=(pat, fdesc) -> match(pat, fdesc).match

# ╔═╡ 363dc920-25d6-11eb-37d7-b5ade368658f
#hide
#TODO:not used as of now
begin
pat4func=r"[a-zA-Z]+\([^\)]*\)(\.[^\)]*\))?"
pat4anonymfunc = r"\([^\)]*\)(\.[^\)]*\))?"
pat4funcargs=r"\([^\)]*\)(\.[^\)]*\))?"
end

# ╔═╡ 9bf47390-25d6-11eb-1c3a-1d305aeb1c06
#hide
fstr = string(methods(grabFuncSig).ms[1])

# ╔═╡ 19ab31e0-4060-11eb-0417-131e3a1f5a5e
md"## Functiondocs"

# ╔═╡ 34ff4880-2b22-11eb-0eef-9bc7ab1aef8f
#export
begin
"""
> struct Functiondocs--> Stores the document of different objects.
> * funcdocs--> Array of strings.
"""
mutable struct Functiondocs
	funcdocs::Array{String, 1}
end

"""
> Functiondocs(funcdocs)--> Helper for accessing the FunctionDocs constructer.
"""
Functiondocs(funcdocs)=Functiondocs(funcdocs)
end

# ╔═╡ d75486f0-2022-11eb-2d95-aded3418c079
#export
begin

"""
> stitchcode(cell::Cell)--> Stitches the code in a Pluto notebook cell with the output of that code. The output is a code block.
"""
function stitchcode(cell::AbstractArray)
	op=' '
	if length(cell) > 1
	    op=values(cell[2])
	end
	string("```","\n$(cell[1])\n","------\nOutput\n------\n","$(op)\n", "```\n")
end

"""
> stitchcode(cellop::AbstractString)--> Removes the quotes from a string and creates a code block with that string inside the newely formed code block
"""
function stitchcode(cellop::AbstractString)
	cleanedop=Export.strip(Export.strip(cellop,"\""), "\"")
	string("```","\n$cleanedop\n","```\n")
end

"""
> stitchcode(fdocs::Functiondocs)--> When supplied with a FunctionDocs type, stitchCode appends together the object docstrings and generates documentation for that particular object
"""
function stitchcode(fdocs::Functiondocs)
		funcdocs=""

		for fdoc in fdocs.funcdocs
		    funcdocs=string(funcdocs, "$(fdoc)\n\n")
	    end

		funcdocs
end
end

# ╔═╡ 931b9ab2-4060-11eb-219d-49b05ff3ca72
md"####  Example"

# ╔═╡ 4cd1f510-2b23-11eb-3051-072d6fb4e81c
begin
funcdocs=Functiondocs(["i", "j"])
funcdocs.funcdocs
end

# ╔═╡ 95219eb0-3e01-11eb-28d2-af58c55dfbd1
#hide
docs=@doc stitchcode

# ╔═╡ 83214680-3eb9-11eb-32bd-01e55390224e
#hide
"$(docs.meta[:results][1].object)"

# ╔═╡ 85446eb5-b22f-41f9-bb9c-be41a7479866
md"## collectfuncdocs"

# ╔═╡ e32bff29-cb7a-40b2-a512-e58f6143d506
#hide
search_url = "https://github-link.vercel.app/api?ghUrl=https://github.com/sapal6/Nbdev.jl/blob/master/src/Documenter.jl&q=Section"

# ╔═╡ d9a872f2-74d0-4f49-8505-b44f03dce3b9
#hide
pattern = r"\\(.+\\)*(.+)(\.jl)"

# ╔═╡ 3f171660-3ec1-11eb-0983-2789adeab1c3
#export
"""
> collectfuncdocs(obj)--> Collects objects (functions, methods, macro structs etc.) and creates an array of documents (generated from the docstrings). Creates aFunctionDocs type from these documents.
"""
function collectfuncdocs(obj)
	docs=doc(obj)
	searchurl = "https://github-link.vercel.app/api?ghUrl="
	giturl = Common.getsettings(joinpath("..", "Project.toml"), ["settings", "github_url"])

	pattern = r"\\(.+\\)*(.+)(\.jl)"
	fn = uppercasefirst(Export.strip(basename(join(match(pattern, string(first(methods(obj)))).captures)), r"[0-9_]"))

	searchurl = "$searchurl$giturl/blob/master/src/$fn&q=$(string(nameof(obj)))"
    fdocs=["$(docs.meta[:results][i].object) [source]($searchurl)" for i=1:length(docs.meta[:results])]
	Functiondocs(fdocs)
end

# ╔═╡ 0a4f347d-dd6b-493b-8473-249270209160
#hide
#TODO: use regex to extract filename from this
uppercasefirst(Export.strip(basename(join(match(pattern, string(first(methods(collectfuncdocs)))).captures)), r"[0-9_]"))

# ╔═╡ 94812075-ec9b-43da-97b6-80ca731fa0d5
#hide
string(first(methods(collectfuncdocs)))

# ╔═╡ 01a22122-4061-11eb-393e-17c15f09e58d
md"#### Example"

# ╔═╡ 89db4130-3ec1-11eb-23ee-eff6d23c1588
collectfuncdocs(stitchcode).funcdocs

# ╔═╡ db32b16e-4061-11eb-23f0-7fdeaab0d0c8
md"## Higher Level API"

# ╔═╡ 0661fdb0-4062-11eb-09d0-030a43180a2c
md"These higher level APIs can either be used directly or are already used internally by nbdev. One such functions which can be used directly is..."

# ╔═╡ df2e8540-4063-11eb-2266-7f423f03bd67
md"## showdoc"

# ╔═╡ a310902e-2b28-11eb-0455-add7ff7c8d6e
#export
begin

"""
> showdoc(o)--> Takes an object and builds markdown documentation.
"""
function showdoc(o)
	docs=collectfuncdocs(o)
	stitchcode(docs)
end
end

# ╔═╡ 0d10aed0-3f9b-11eb-1bcd-dbdb5e5068f4
showdoc(line)

# ╔═╡ 87c22750-3f94-11eb-201c-a3c6374881f4
showdoc(Page)

# ╔═╡ 41b87320-3f9b-11eb-1ed4-fdfefdc01627
showdoc(sections)

# ╔═╡ 48beb2b0-3f9b-11eb-2756-919a82c5c0de
showdoc(name)

# ╔═╡ a13c23a3-f9c1-41dd-850b-b1e99a80c26f
showdoc(runandupdatenb)

# ╔═╡ 59aca6d0-405f-11eb-2252-633f4d0ccdbc
showdoc(stitchcode)

# ╔═╡ bb865340-4061-11eb-065c-030bf004197f
showdoc(Functiondocs)

# ╔═╡ d038c980-4061-11eb-19a5-5bab5b196788
showdoc(collectfuncdocs)

# ╔═╡ f17bc160-2e46-11eb-0d65-cf6185b4f406
showdoc(showdoc)

# ╔═╡ 2b104160-4e83-11eb-1a78-a96f77e1aff4
#noop
begin
"""
> An example of a struct
"""
struct Mystruct
	name
end
end

# ╔═╡ 449bb6f2-4e83-11eb-27ae-298352285e98
mystruct=Mystruct("test")

# ╔═╡ df1998f0-4ea1-11eb-0626-dbead118373f
typeof(mystruct)

# ╔═╡ 26e54202-9cc7-4b8f-b5b4-d83f72547dbc
md"`showdoc(Mystruct)`"

# ╔═╡ e1d02746-3098-4832-af6c-81f99fab6ca5
showdoc(Mystruct)

# ╔═╡ 10866060-5980-11eb-0422-e7300713c6a4
md"Currently nbdev is unable to recognize the docstrings of inline expressions.👇"

# ╔═╡ 4fee8610-5980-11eb-137a-83f4aa64e933
#noop
"> inlinetest--> This is a inline test expression"
inlinetest=str->replace(str, "1"=> "one")

# ╔═╡ 83bb0590-5980-11eb-3159-b376954405ef
#hide
#showDoc(inlinetest)

# ╔═╡ 285cf4e0-4064-11eb-3162-1b399c464a1a
md"## createpage"

# ╔═╡ 36b846d0-2024-11eb-3784-89a02343cd0b
#export
begin
function maintainheading(str:: AbstractString)
	res= nothing
	if startswith(str, """

""")
			res = Export.strip(Export.strip(Export.strip(str, """

"""), """
"""), """
""")
			res = string("## ", res)
	else
		res = str
	end
	res
end

"""
> CreatePage--> Creates the Page type from the markdown and example code cells of the supplied notebook. The filename is the name of the notebook which is parsed.
"""
function createpage(fn::AbstractString, nb::Notebook)
	sections=Section[]
	res=nothing

	for cell in nb.cells

		if cell.errored
			error("Build stopped. Seems like the code $cell.code has an error")
			break
	    end
	    if startswith(cell.code, "md")
			cleanop = maintainheading(cell.output.body)
			push!(sections, Section(cleanop))
		elseif !startswith(cell.code, "#export") && !startswith(cell.code, "#hide")
			if occursin( "showdoc", cell.code)
				cleanedop=Export.strip(cell.output.body, "\"")
				cleanedop=replace(cleanedop, "\\n"=>"\n")
				push!(sections, Section(cleanedop))
			elseif occursin( "#noop", cell.code)
				stitchedcode=stitchcode([cell.code])
			    push!(sections, Section(stitchedcode))
			else
				stitchedcode=stitchcode([cell.code, cell.output.body])
			    push!(sections, Section(stitchedcode))
			end
		end
	end
	if !isempty(sections)
	    res=Page(sections, basename(fn))
	end

	return res

end
end

# ╔═╡ 7216e720-411e-11eb-1103-19bf4993ef1e
showdoc(createpage)

# ╔═╡ 7f938250-411e-11eb-0d30-b53cf2c8bc97
md"While generating document you don't need to call this function. This is done automatically😃 for you when nbdev generates documents."

# ╔═╡ 8d7b5280-28a0-11eb-282d-2dbf124460da
#hide
begin
const _header = ""
const _footer = ""
end

# ╔═╡ 7b6fd3d0-411f-11eb-3786-ff38ee7d0291
md"## savepage"

# ╔═╡ 4c5c7c22-28a0-11eb-0069-cb78e0e7e0ee
#export
begin

"""
> savepage(io, page::Page)--> Take the contents from a Page type and write to the io
"""
function savepage(io, page::Page)

	pageheading=uppercasefirst(Export.strip(Export.strip(page.name, r"[0-9_]"), r".jl"))

    println(io, "
$pageheading
")

	for section in page.sections
			println(io, section.line)
    end

end

"""
> savepage(page::Page, path::String)--> Given a Page type and the required path, creates the related markdwon file in the specified path. The name of the resulting markdown file is same as the nameof the notebook for which the document is generated
"""
function savepage(page::Page, path::String)
	fn=lowercase(Export.strip(Export.strip(page.name, r"[0-9_]"), r".jl"))
	open(joinpath(path, fn*".md"), "w") do io
        savepage(io, page)
    end
end
end

# ╔═╡ 870d3240-4120-11eb-0dca-89337e801493
showdoc(savepage)

# ╔═╡ 943f2fe2-4120-11eb-11a9-7785d11d3c36
md"Nbdev calls the required method of `save_page` automatically during document generation."

# ╔═╡ ed8b55f0-4121-11eb-1a2b-a77bea8bfe7f
md"## export2md"

# ╔═╡ f31331e0-28c2-11eb-1014-95ed88d77469
#export
begin
"""
> export2md(fn::String, path::String)--> Generate document for a file in the given path
"""
function export2md(fn::String, path::String)
	nb=runandupdatenb(fn)
	page=createpage(fn, nb)
	if page != nothing
	    savepage(page, path)
	end
end

"""
> export2md(fns::AbstractVector, path::String)--> Map the `export2md(file, path)` to a given vector of file.
"""
function export2md(fns::AbstractVector, path::String)
	n = length(fns)
	p = Progress(n, 1)
	for fn in fns
	  next!(p)
      export2md(fn, path)
	end
end

"""
> export2md()--> Higher level API to generate documents for all the valid notebooks
"""
export2md(nbsdir)=export2md(Export.readfilenames(joinpath(pwd(), nbsdir)), "docs")
end

# ╔═╡ 7081aafd-d6ef-4672-99b0-6080497e2498
#export
export showdoc, export2md

# ╔═╡ 807db3e2-4121-11eb-136d-ad470b83a46f
showdoc(export2md)

# ╔═╡ 0bf8a871-822a-4281-9186-2355751451d4
Export.readfilenames(joinpath(pwd()))

# ╔═╡ 8c376960-4121-11eb-1627-cf0d01bcf47b
md"The `export2md()` is what gets summoned when document generation is invoked. Like most things in nbdev (and unlike most things in life) this too gets invoked automatically. 🥳"

# ╔═╡ f69f6cba-4fa3-447c-9f8e-f9c5d1f8f7d5
#### export2readme

# ╔═╡ 15c1479b-8202-4d3c-8bdd-903c3c43775d
#hide
#joinpath(pwd(), "nbs/*index.jl")

# ╔═╡ 9c07bd79-f47e-4897-830b-63748f674335
#export
begin
"""
> export2readme()--> create readme from the contents of Index notebook
"""
function export2readme()
    cp(normpath(joinpath(".", "docs/index.md")), normpath(joinpath(".", "README.md")), force=true)
end
end

# ╔═╡ 1d1e1c1f-75dc-4a72-ae01-9bc337afe14b
showdoc(export2readme)

# ╔═╡ 58b6fa50-0ba8-11eb-1ccf-1328cbe524b4
#hide
Export.notebook2script(joinpath("..", "nbs"), joinpath("..", "src"))

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
PlutoTest = "cb4044da-4d16-4ffa-a6a3-8cad7f73ebdc"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[compat]
Pluto = "~0.16.1"
PlutoTest = "~0.2.0"
PlutoUI = "~0.7.10"
ProgressMeter = "~1.7.1"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Configurations]]
deps = ["ExproniconLite", "OrderedCollections", "TOML"]
git-tree-sha1 = "41d153a50b001a7c534f19e263540cca1a4e7cf3"
uuid = "5218b696-f38b-4ac9-8b61-a12ec717816d"
version = "0.16.4"

[[DataAPI]]
git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.9.0"

[[DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[ExproniconLite]]
git-tree-sha1 = "45b421f664eba4cd4bed8e5e706f69f2ccd8fcb3"
uuid = "55351af7-c7e9-48d6-89ff-24e801d99491"
version = "0.6.10"

[[FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"

[[FuzzyCompletions]]
deps = ["REPL"]
git-tree-sha1 = "2cc2791b324e8ed387a91d7226d17be754e9de61"
uuid = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2"
version = "0.4.3"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
git-tree-sha1 = "60ed5f1643927479f845b0135bb369b031b541fa"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.9.14"

[[HypertextLiteral]]
git-tree-sha1 = "72053798e1be56026b81d4e2682dbe58922e5ec9"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.0"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.3"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[MsgPack]]
deps = ["Serialization"]
git-tree-sha1 = "a8cbf066b54d793b9a48c5daa5d586cf2b5bd43d"
uuid = "99f44e22-a591-53d1-9472-aa23ef4bd671"
version = "1.1.0"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "9d8c00ef7a8d110787ff6f170579846f776133a9"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.4"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Pluto]]
deps = ["Base64", "Configurations", "Dates", "Distributed", "FileWatching", "FuzzyCompletions", "HTTP", "InteractiveUtils", "Logging", "Markdown", "MsgPack", "Pkg", "REPL", "Sockets", "TableIOInterface", "Tables", "UUIDs"]
git-tree-sha1 = "d4955d6b5267ed826d03e3f05cc23426492b23c9"
uuid = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
version = "0.16.1"

[[PlutoTest]]
deps = ["HypertextLiteral", "InteractiveUtils", "Markdown", "Test"]
git-tree-sha1 = "92b8ae1eee37c1b8f70d3a8fb6c3f2d81809a1c5"
uuid = "cb4044da-4d16-4ffa-a6a3-8cad7f73ebdc"
version = "0.2.0"

[[PlutoUI]]
deps = ["Base64", "Dates", "HypertextLiteral", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "Suppressor"]
git-tree-sha1 = "26b4d16873562469a0a1e6ae41d90dec9e51286d"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.10"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[ProgressMeter]]
deps = ["Distributed", "Printf"]
git-tree-sha1 = "afadeba63d90ff223a6a48d2009434ecee2ec9e8"
uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
version = "1.7.1"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[Suppressor]]
git-tree-sha1 = "a819d77f31f83e5792a76081eee1ea6342ab8787"
uuid = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
version = "0.2.0"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[TableIOInterface]]
git-tree-sha1 = "9a0d3ab8afd14f33a35af7391491ff3104401a35"
uuid = "d1efa939-5518-4425-949f-ab857e148477"
version = "0.1.6"

[[TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
git-tree-sha1 = "1162ce4a6c4b7e31e0e6b14486a6986951c73be9"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.5.2"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[URIs]]
git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
version = "1.3.0"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╠═4cb4aa50-3e01-11eb-3460-5f109773492b
# ╠═6c0aeda1-81d7-469d-8443-bbf347feede6
# ╠═151ec8b0-2b27-11eb-1ec2-a7c5e4c13db9
# ╠═07b9d565-4690-4e48-bcc2-05428b665ca1
# ╠═77a9e510-4ae2-44a2-9536-17a89a54d6f6
# ╠═b068dfd2-0eb3-11eb-109a-d1b6ef1eeca0
# ╠═1d83078e-2024-11eb-0e5f-51310d134662
# ╠═27ff1d70-1201-11eb-2003-27cb52571be6
# ╠═2bc37e16-7ff5-430b-b29c-571db5e9b6b5
# ╠═295d813f-1e85-4b2e-a953-1bd0087a85e2
# ╠═9930ad11-84da-45e0-95dc-f46c4d48174a
# ╠═fcfaccaf-cc88-44bb-8971-c1328d8599ee
# ╠═7081aafd-d6ef-4672-99b0-6080497e2498
# ╠═25ff264e-3ec5-11eb-362c-07b4e24c635a
# ╠═28951cc5-0f45-48fc-965a-4257de0332fa
# ╠═c371d5ec-8d89-4f33-8b6d-9add8e15938f
# ╠═c099d54a-38df-46ee-8fb1-37c2b90c1df2
# ╠═bf4e47f0-3ec5-11eb-1b65-5fe2e7a88ff1
# ╠═5a2e9790-201f-11eb-0df4-f90b3cc54f20
# ╠═0d10aed0-3f9b-11eb-1bcd-dbdb5e5068f4
# ╠═f1318ca0-3f97-11eb-3727-3f71064c77bf
# ╠═cfb6d710-3f97-11eb-31ed-6daa12cf592e
# ╠═610d9263-645a-46aa-8f16-12e90787b7bc
# ╠═309585e2-3f93-11eb-0873-d9b0d7a6200e
# ╠═28a4e100-17ac-11eb-172c-2d0e73460caa
# ╠═87c22750-3f94-11eb-201c-a3c6374881f4
# ╠═41b87320-3f9b-11eb-1ed4-fdfefdc01627
# ╠═48beb2b0-3f9b-11eb-2756-919a82c5c0de
# ╠═997a4950-3f94-11eb-2e66-d9a70cc175d1
# ╠═ea16dab0-3f97-11eb-19b5-0f4998ba3e39
# ╠═11057000-3f98-11eb-27aa-ffb097ca25ed
# ╠═7e959872-421d-11eb-3784-2f110840a628
# ╠═dd721172-4443-11eb-3833-3f74ff47dd42
# ╠═8d851420-08af-11eb-26b0-63501c72011a
# ╠═9478dd20-08af-11eb-1663-a7db573d2187
# ╠═74879c7e-0eeb-11eb-31bf-251621d154a6
# ╠═e2952860-1ade-11eb-20ca-091a45fab2f2
# ╠═d9fffed0-2f3f-11eb-16b4-4b2778f792d9
# ╠═2fc24ff0-11f7-11eb-18a8-1b2b989fa189
# ╠═743238d0-1918-11eb-3dfc-6f30db92923c
# ╠═8691e572-1918-11eb-011c-639d3617e076
# ╠═5001a5b0-11ff-11eb-054a-6921da78afa3
# ╠═a13c23a3-f9c1-41dd-850b-b1e99a80c26f
# ╠═f1d7ed22-1f8a-11eb-035d-6de2cb48bb8b
# ╠═6b9cb0f0-1f5e-11eb-1e15-9f0c8295b59f
# ╠═c3d2cf20-1f8b-11eb-0381-01270b1494b3
# ╠═1d71bb00-1f8b-11eb-31de-69aebd625201
# ╠═bda9c5a0-1f8a-11eb-396b-97f97add91db
# ╠═d75486f0-2022-11eb-2d95-aded3418c079
# ╠═59aca6d0-405f-11eb-2252-633f4d0ccdbc
# ╠═00989200-25d6-11eb-3139-8dd2ca0346f8
# ╠═363dc920-25d6-11eb-37d7-b5ade368658f
# ╠═9bf47390-25d6-11eb-1c3a-1d305aeb1c06
# ╠═19ab31e0-4060-11eb-0417-131e3a1f5a5e
# ╠═34ff4880-2b22-11eb-0eef-9bc7ab1aef8f
# ╠═bb865340-4061-11eb-065c-030bf004197f
# ╠═931b9ab2-4060-11eb-219d-49b05ff3ca72
# ╠═4cd1f510-2b23-11eb-3051-072d6fb4e81c
# ╠═95219eb0-3e01-11eb-28d2-af58c55dfbd1
# ╠═83214680-3eb9-11eb-32bd-01e55390224e
# ╠═85446eb5-b22f-41f9-bb9c-be41a7479866
# ╠═e32bff29-cb7a-40b2-a512-e58f6143d506
# ╠═d9a872f2-74d0-4f49-8505-b44f03dce3b9
# ╠═0a4f347d-dd6b-493b-8473-249270209160
# ╠═94812075-ec9b-43da-97b6-80ca731fa0d5
# ╠═3f171660-3ec1-11eb-0983-2789adeab1c3
# ╠═d038c980-4061-11eb-19a5-5bab5b196788
# ╠═01a22122-4061-11eb-393e-17c15f09e58d
# ╠═89db4130-3ec1-11eb-23ee-eff6d23c1588
# ╟─db32b16e-4061-11eb-23f0-7fdeaab0d0c8
# ╠═0661fdb0-4062-11eb-09d0-030a43180a2c
# ╠═df2e8540-4063-11eb-2266-7f423f03bd67
# ╠═a310902e-2b28-11eb-0455-add7ff7c8d6e
# ╠═f17bc160-2e46-11eb-0d65-cf6185b4f406
# ╠═2b104160-4e83-11eb-1a78-a96f77e1aff4
# ╠═449bb6f2-4e83-11eb-27ae-298352285e98
# ╠═df1998f0-4ea1-11eb-0626-dbead118373f
# ╠═26e54202-9cc7-4b8f-b5b4-d83f72547dbc
# ╠═e1d02746-3098-4832-af6c-81f99fab6ca5
# ╠═10866060-5980-11eb-0422-e7300713c6a4
# ╠═4fee8610-5980-11eb-137a-83f4aa64e933
# ╠═83bb0590-5980-11eb-3159-b376954405ef
# ╠═285cf4e0-4064-11eb-3162-1b399c464a1a
# ╠═36b846d0-2024-11eb-3784-89a02343cd0b
# ╠═7216e720-411e-11eb-1103-19bf4993ef1e
# ╠═7f938250-411e-11eb-0d30-b53cf2c8bc97
# ╠═8d7b5280-28a0-11eb-282d-2dbf124460da
# ╠═7b6fd3d0-411f-11eb-3786-ff38ee7d0291
# ╠═4c5c7c22-28a0-11eb-0069-cb78e0e7e0ee
# ╠═870d3240-4120-11eb-0dca-89337e801493
# ╠═943f2fe2-4120-11eb-11a9-7785d11d3c36
# ╠═ed8b55f0-4121-11eb-1a2b-a77bea8bfe7f
# ╠═f31331e0-28c2-11eb-1014-95ed88d77469
# ╠═807db3e2-4121-11eb-136d-ad470b83a46f
# ╠═0bf8a871-822a-4281-9186-2355751451d4
# ╠═8c376960-4121-11eb-1627-cf0d01bcf47b
# ╠═f69f6cba-4fa3-447c-9f8e-f9c5d1f8f7d5
# ╠═15c1479b-8202-4d3c-8bdd-903c3c43775d
# ╠═9c07bd79-f47e-4897-830b-63748f674335
# ╠═1d1e1c1f-75dc-4a72-ae01-9bc337afe14b
# ╠═58b6fa50-0ba8-11eb-1ccf-1328cbe524b4
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


text:
@require DataFrames begin

@generated function _filldf(columns, enumerable)
    n = length(columns.types)
    push_exprs = Expr(:block)
    for i in 1:n
        if columns.parameters[i] <: DataArray
            ex = :( push!(columns[$i], isnull(i[$i]) ? NA : get(i[$i])) )
        else
            ex = :( push!(columns[$i], i[$i]) )
        end
        push!(push_exprs.args, ex)
    end

    quote
        for i in enumerable
            $push_exprs
        end
    end
end

function collect{T<:NamedTuple}(enumerable::Enumerable{T}, ::Type{DataFrames.DataFrame})
    columns = []
    for t in T.types
        if isa(t, TypeVar)
            push!(columns, Array(Any,0))
        elseif t <: Nullable
            push!(columns, DataArray(t.parameters[1],0))
        else
            push!(columns, Array(t,0))
        end
    end
    df = DataFrames.DataFrame(columns, fieldnames(T))
    _filldf((df.columns...), enumerable)
    return df
end

function collect{TS,Provider}(source::Queryable{TS,Provider}, ::Type{DataFrames.DataFrame})
    collect(query(collect(source)), DataFrames.DataFrame)
end

end


text:
# name = "Alpine"
# Alpine = "07493b3f-dabb-5b16-a503-4139292d7dd4"

using Alpine, Ipopt, Cbc

const ALP_IPOPT = optimizer_with_attributes(Ipopt.Optimizer, MOI.Silent() => true, "sb" => "yes")
const ALP_CBC = optimizer_with_attributes(Cbc.Optimizer, MOI.Silent() => true)

# ALP_OPT = optimizer_with_attributes(Alpine.Optimizer, "nlp_solver" => ALP_IPOPT,
# "mip_solver" => ALP_CBC,
# "loglevel" => 100)
# ALP = ALP_OPT

ALP_OPT = Alpine.Optimizer()
MOI.set(ALP_OPT, MOI.RawParameter("nlp_solver"), ALP_IPOPT)
MOI.set(ALP_OPT, MOI.RawParameter("mip_solver"), ALP_CBC)
ALP = MOI.Bridges.Constraint.SOCtoNonConvexQuad{Float64}(ALP_OPT)

push!(solvers_nlp_lowtol, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-5)))
push!(solvers_nlp, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-9)))
push!(solvers_nlp, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-9, with_slack = true)))

push!(solvers, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-9)))
push!(solvers_cached, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-9)))
push!(solvers_quad, (opt = ALP, mode = BilevelJuMP.ProductMode(1e-9)))

push!(solvers_nlp_sd, (opt = ALP, mode = BilevelJuMP.StrongDualityMode(1e-9, inequality = true)))
push!(solvers_nlp_sd, (opt = ALP, mode = BilevelJuMP.StrongDualityMode(inequality = false)))
push!(solvers_nlp_sd_i, (opt = ALP, mode = BilevelJuMP.StrongDualityMode(1e-9, inequality = true)))
push!(solvers_nlp_sd_e, (opt = ALP, mode = BilevelJuMP.StrongDualityMode(inequality = false)))


text:
"""
`assemble_system!` - assemble the linear system for solving
"""
function assemble_system!(problem::FDPoisson2DProblem{TF, TBC, TBC, TBC, TBC}) where {TF<:AbstractArray, TBC<:DirichletBC}

    N = problem.nx * problem.ny; # get system size

    for k in 1:N
        # get local coordiantes from global index
        # k = i + (j-1) * nx
        j, i = fldmod1(k, problem.nx);
        # process corners, then edges, then interior
        if (i==1 && j==1)
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx ]  = -1/problem.Δy^2;
        elseif (i==1 && j == problem.ny)
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2
        elseif (i==problem.nx && j==1)
            problem.A[k,k-1] =  -1/problem.Δx^2;
            problem.A[k,k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k,k+problem.nx] =  -1/problem.Δy^2;
        elseif (i==problem.nx && j ==problem.ny)
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
        elseif(i==1 && j>1 && j <problem.ny)
            # left edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx] = -1/problem.Δy^2;
        elseif(i==problem.nx  && j>1 && j <problem.ny)
            # right edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k + problem.nx] = -1/problem.Δy^2;
        elseif(i>1 && i<problem.nx && j==1)
            # bottom edge
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k + problem.nx] = -1/problem.Δy^2;
        elseif(i>1 && i<problem.nx && j==problem.ny)
            # top edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] =-1/problem.Δx^2;
            problem.A[k, k] =  2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
        else
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx] = -1/problem.Δy^2;
        end
    end
    @. problem.rhs = problem.f[:];
    problem

end

"""
`assemble_system!` - assemble the linear system for solving
"""
function assemble_system!(problem::FDPoisson2DProblem{TF, TBC, TBC, TBC, TBC}) where {TF<:Function, TBC<:DirichletBC}

    N = problem.nx * problem.ny; # get system size

    for k in 1:N
        # get local coordiantes from global index
        # k = i + (j-1) * nx
        j, i = fldmod1(k, problem.nx);
        # process corners, then edges, then interior
        if (i==1 && j==1)
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx ]  = -1/problem.Δy^2;
        elseif (i==1 && j == problem.ny)
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2
        elseif (i==problem.nx && j==1)
            problem.A[k,k-1] =  -1/problem.Δx^2;
            problem.A[k,k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k,k+problem.nx] =  -1/problem.Δy^2;
        elseif (i==problem.nx && j ==problem.ny)
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
        elseif(i==1 && j>1 && j <problem.ny)
            # left edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx] = -1/problem.Δy^2;
        elseif(i==problem.nx  && j>1 && j <problem.ny)
            # right edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k + problem.nx] = -1/problem.Δy^2;
        elseif(i>1 && i<problem.nx && j==1)
            # bottom edge
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k + problem.nx] = -1/problem.Δy^2;
        elseif(i>1 && i<problem.nx && j==problem.ny)
            # top edge
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] =-1/problem.Δx^2;
            problem.A[k, k] =  2/problem.Δx^2 + 2/problem.Δy^2;
            problem.A[k, k+1] = -1/problem.Δx^2;
        else
            problem.A[k, k-problem.nx] = -1/problem.Δy^2;
            problem.A[k, k-1] = -1/problem.Δx^2;
            problem.A[k, k] = 2/problem.Δx^2 + 2/problem.Δy^2;;
            problem.A[k, k+1] = -1/problem.Δx^2;
            problem.A[k, k+problem.nx] = -1/problem.Δy^2;
        end
        problem.rhs[k] = problem.f(problem.x[i], problem.y[j]);
    end
    problem

end


text:
using Test, CSV, DelimitedFiles, ModelSelection.PreliminarySelection, ModelSelection.Preprocessing

data_fat = CSV.read(DATABASE_FAT)

@testset "Lasso" begin
    @testset "Fat dataset" begin
        data = Preprocessing.input("y *", data_fat)
        data = PreliminarySelection.lasso(data)
        @test length(data.expvars, 1) < length(data.expvars)
    end
end


text:
#
# builder.jl --
#
# Methods to parse OI-FITS extension definitions and to build data-block
# instances.
#
#------------------------------------------------------------------------------

module Builder

using Compat

using ..OIFITS
using ..OIFITS:
    OIData,
    contents,
    convert_float_type,
    convert_key_type,
    fix_name,
    get_extname,
    is_attached,
    is_float,
    is_integer,
    is_logical,
    is_string,
    to_fieldname,
    to_float,
    to_integer,
    to_logical,
    to_string

"""

Type `OIFITS.Builder.FieldDefinition` is used to store the definition of a
keyword/column field.

"""
struct FieldDefinition
    name::String     # Keyword/column name as a string.
    symb::Symbol     # Keyword/column symbolic name.
    iskeyword::Bool  # Is keyword? (otherwise column)
    isoptional::Bool # Optional field?
    multiplier::Int  # Multiplier: 1 for keywords, number of cells for columns
                     # (a negative number -N means an array of N dimensions
                     # each equal to the number of spectral channels.
    type::Symbol     # Data type.
    units::String    # Units.
    descr::String    # Description.
end

"""

Type `OIFITS.Builder.DataBlockDefinition` is used to store the definition of
data-block.

"""
struct DataBlockDefinition
    extname::String                    # name of OI-FITS extension
    revn::Int                          # revision of OI-FITS extension
    fields::Vector{Symbol}             # ordered symbolic names of fields
    spec::Dict{Symbol,FieldDefinition} # dictionary of field specifications
    function DataBlockDefinition(extname::AbstractString, revn::Integer,
                                 defs::Vector{FieldDefinition})
        @assert revn ≥ 1
        spec = Dict{Symbol,FieldDefinition}()
        fields = Array{Symbol}(undef, length(defs))
        @inbounds for j in 1:length(defs)
            def = defs[j]
            fields[j] = def.symb
            spec[def.symb] = def
        end
        new(extname, revn, fields, spec)
    end
end

OIFITS.get_extname(def::DataBlockDefinition) = def.extname

# FIELDS is a dictionary indexed by the extension name (e,g., "OI_VIS"), each
# entry stores a set of its fields.
const FIELDS = Dict{String,Set{Symbol}}()

# REVISIONS is a dictionary indexed by the extension name (e,g., "OI_VIS") and
# whose value is the last revision number.
const REVISIONS = Dict{String,Int}()

# FORMATS table is indexed by a 2-tuple composed by the datablock name
# and by the revision number of the corresponding OI-FITS table.
const FORMATS = Dict{Tuple{String,Int},DataBlockDefinition}()
format_key(name::AbstractString, revn::Integer) =
    (to_string(name), to_integer(revn))

"""
    OIFITS.Builder.last_revision(arg) -> revn

yields the last revision number of OI-FITS extension given its name, a
data-block type or a data-block instance.  The value `0` is returned if `arg`
is not a known OI-FITS extension name.

"""
last_revision(db::OIDataBlock) = last_revision(db.extname)
last_revision(::Type{T}) where {T<:OIDataBlock} = last_revision(get_extname(T))
last_revision(extname::String) = get(REVISIONS, extname, 0)

"""
    OIFITS.Builder.initialize!(obj, dict) -> obj

set as many fields as possible to zero in structured object `obj`, then
populates the fields of the structured object `obj` with the key-value pairs in
`dict`, finally returns `obj`.  This method is intended for freshly created
objects whose fields have not yet been initialized.  If key `:revn` is
undefined in `dict`, the last revision of the format is assumed.

"""
function initialize!(obj::T, dict::AbstractDict) where {T<:OIDataBlock}
    clear_fields!(obj)
    if haskey(dict, :revn) == false
        obj.revn = last_revision(T)
    end
    length(dict) > 0 && populate!(obj, dict)
    return obj
end

"""
    OIFITS.Builder.populate!(obj, dict) -> obj

populates the fields of the structured object `obj` with the key-value pairs
in `dict` and returns `obj`.

"""
function populate!(obj::T, dict::AbstractDict) where {T}
    @assert isstructtype(T)
    for (key, val) in dict
        sym = Symbol(key)
        setproperty!(obj, sym, val)
    end
    return obj
end

"""
    OIFITS.Builder.clear_fields!(obj) -> obj

set as many fields as possible to zero in structured object `obj` and returns
`obj`.

"""
function clear_fields!(obj::T) where {T}
    @assert isstructtype(T)
    for sym in fieldnames(T)
        clear_field!(obj, sym)
    end
    return obj
end

"""
    OIFITS.Builder.clear_field!(obj, sym)

set field `sym` in structured object `obj` to zero if that makes sense, that is
if the fields is a number.

"""
clear_field!(obj::T, sym::Symbol) where {T} =
    (_clear_field!(obj, sym, fieldtype(T, sym)); nothing)

_clear_field!(obj, sym::Symbol, ::Type{T}) where {T<:Number} =
    setfield!(obj, sym, zero(T))

_clear_field!(obj, sym::Symbol, ::Type{<:Any}) =
    nothing

# Extend Base.push: to add data-blocks to
Base.push!(obj::OIMaster) = obj

Base.push!(obj::OIMaster, others::OIDataBlock...) = push!(obj, others)

function Base.push!(obj::OIMaster{T},
                    others::Union{AbstractVector{<:OIDataBlock},
                                Tuple{Vararg{OIDataBlock}}}) where {T}
    # First push each new data-block.  Second, update links of all stored
    # data-blocks.
    for i in eachindex(others)
        _push!(obj, convert_float_type(T, others[i]))
    end
    _update_links!(obj)
end

# Push nothing.
_push!(master::OIMaster, ::Nothing) = nothing

# Just push one data-block.
function _push!(master::OIMaster{T}, db::OIDataBlock{T}) where {T}
    # Do nothing if data-block already attached to master.
    is_attached(db, master) && return

    # Consider special cases.
    if isa(db, OITarget)
        isdefined(master, :target) &&
            error("only one OI_TARGET data-block can be attached")
        setfield!(master, :target, db)
    elseif isa(db, OIWavelength)
        insname = fix_name(db.insname)
        haskey(master.instr, insname) && error("master already have an ",
                                               "OI_WAVELENGTH data-block with ",
                                               "INSNAME=\"", insname, "\"")
        master.instr[insname] = db
    elseif isa(db, OIArray)
        arrname = fix_name(db.arrname)
        haskey(master.array, arrname) && error("master already have an ",
                                               "OI_ARRAY data-block with ",
                                               "ARRNAME=\"", arrname, "\"")
        master.array[arrname] = db
    elseif isa(db, OICorrelation)
        corrname = fix_name(db.corrname)
        haskey(master.correl, corrname) && error("master already have an ",
                                                 "OI_CORR data-block with ",
                                                 "CORRNAME=\"", corrname, "\"")
        master.correl[corrname] = db
    end

    # Push data-block to the list of data-blocks stored by the master.
    if is_attached(db)
        # Make a copy if data-block attached to any (other) master.
        push!(contents(master), copy(db))
    else
        push!(contents(master), db)
    end
    setfield!(db, :owner, master)
    nothing
end

function _update_links!(master::OIMaster)
    for i in eachindex(master)
        _update_links!(master[i])
    end
    master
end

_update_links!(db::OITarget) = nothing
_update_links!(db::OIArray) = nothing
_update_links!(db::OICorrelation) = nothing
_update_links!(db::OIWavelength) = nothing
_update_links!(db::OIPolarization) = begin
    _link_array!(db)
    _link_instr!(db)
end
_update_links!(db::OIData) = begin
    _link_array!(db)
    _link_instr!(db)
    _link_correl!(db)
end

_link_array!(db::OIDataBlock) = _link_field!(db, :arrname, :array)
_link_instr!(db::OIDataBlock) = _link_field!(db, :insname, :instr)
_link_correl!(db::OIDataBlock) = _link_field!(db, :corrname, :correl)

function _link_field!(db::OIDataBlock, from::Symbol, to::Symbol)
    if (is_attached(db) && (name = getproperty(db, from)) !== nothing &&
        (lnk = get(getfield(db.owner, to), fix_name(name), nothing)) !== nothing)
        setfield!(db, to, lnk)
    end
    nothing
end

"""
     OIFITS.Builder.get_datatype(c)

yields the symbolic identifier of the data type in a FITS table column defined by
the letter `c`.

| Symbol     | Letters        |
|:-----------|:---------------|
| `:LOGICAL` | `'L'`          |
| `:INTEGER` | `'I'` or `'J'` |
| `:REAL`    | `'D'` or `'E'` |
| `:COMPLEX` | `'C'`          |
| `:STRING`  | `'A'`          |
| `:UNKNOWN` | other letters  |

"""
get_datatype(c::Char) =
    # It is about 2 times faster to search with a series of tests instead of
    # using a dictionary.  Converting to uppercase/lowercase is much longer so
    # we keep upper- and lowercase version of each letter.
    (c == 'A' ? :STRING  :
     c == 'C' ? :COMPLEX :
     c == 'D' ? :FLOAT   :
     c == 'E' ? :FLOAT   :
     c == 'I' ? :INTEGER :
     c == 'J' ? :INTEGER :
     c == 'L' ? :LOGICAL :
     c == 'a' ? :STRING  :
     c == 'c' ? :COMPLEX :
     c == 'd' ? :FLOAT   :
     c == 'e' ? :FLOAT   :
     c == 'i' ? :INTEGER :
     c == 'j' ? :INTEGER :
     c == 'l' ? :LOGICAL : :UNKNOWN)

"""
    OIFITS.Builder.get_description(db) -> (extname, revn, defn)

yields the FITS extension name, revision number and format definitions of the
OI-FITS data-block `db`.

"""
get_description(db::OIDataBlock) = (db.extname, db.revn, get_definition(db))

"""
    OIFITS.Builder.get_definition(db)

or

    OIFITS.Builder.get_definition(extname, revn)

yield the format definition for datablock `db` of for the OI-FITS extension
`extname` (e.g., "OI_TARGET") in revision `revn` of OI-FITS standard, throwing
an error if not found.  The returned value is an instance of `DataBlockDefinition`.

    OIFITS.get_definition(extname, revn, def)

is similar but returns `def` if the format is not found.

"""
function get_definition(extname::AbstractString, revn::Integer)
    val = get_definition(extname, revn, nothing)
    val === nothing && error("unknown OI-FITS extension \"", extname,
                             "\" (revision ", revn, ")")
    return val
end

get_definition(extname::AbstractString, revn::Integer, def) =
    get(FORMATS, format_key(extname, revn), def)

get_definition(db::OIDataBlock) = get_definition(db.extname, db.revn)

"""
    OIFITS.Builder.has_member(obj, sym)

yields whether object `obj` has a defined member of symbolic name `sym`.
Object `obj` can be a dictionary with symbolic keys or an `OIDataBlock`
instance.

"""
@inline has_member(dict::AbstractDict{Symbol}, sym::Symbol) = haskey(dict, sym)
@inline has_member(obj::T, sym::Symbol) where {T<:OIDataBlock} =
    hasfield(T, sym) && isdefined(obj, sym)

"""
    OIFITS.Builder.check_contents(obj)

checks whether object `obj` is a valid instance of an OI-FITS data-block.
This method checks that all mandatory members of `obj` are defined and have
consistent size.

    OIFITS.Builder.check_contents(obj, extname, revn)

checks whether the entries/fields of object `obj` are valid for an OI-FITS
extension `extname`, revison `revn`.  Object `obj` can be a dictionary with
symbolic keys or an `OIDataBlock` instance.

FIXME: check number of wavelengths etc. for a linked object.

"""
function check_contents(dict::AbstractDict{<:Union{Symbol,AbstractString}},
                        extname::AbstractString, revn::Integer)
    check_contents(convert_key_type(Symbol, dict),
                   to_string(extname), to_integer(revn))
end

check_contents(obj::OIDataBlock) = check_contents(obj, obj.extname, obj.revn)

function check_contents(obj::Union{AbstractDict{Symbol},OIDataBlock},
                        extname::String, revn::Int)
    def = get_definition(extname, revn)
    rows = -1         # number of rows in the OI-FITS table
    channels = -1     # number of spectral channels
    for (key, val) in obj
        # Ignore fields not in the specifications.
        spec = get(def.spec, key, nothing)
        spec === nothing && continue

        # Check value type.
        if spec.type === :LOGICAL
            is_logical(val) || error("expecting boolean value for `", key,
                                     "` field of OI-FITS extension ", extname)
        elseif spec.type === :INTEGER
            is_integer(val) || error("expecting integer value for `", key,
                                     "` field of OI-FITS extension ", extname)
        elseif spec.type === :FLOAT
            is_float(val) || error("expecting floating-point value for `", key,
                                   "` field of OI-FITS extension ", extname)
        elseif spec.type === :COMPLEX
            is_complex(val) || error("expecting complex value for `", key,
                                     "` field of OI-FITS extension ", extname)
        elseif spec.type === :STRING
            is_string(val) || error("expecting string value for `", key,
                                    "` field of OI-FITS extension ", extname)
        else
            error("unknown value type `:", spec.type, "`")
        end

        # Check value dimensions.
        dims = (isa(val, Array) ? size(val) : ())
        rank = length(dims)
        if spec.iskeyword
            rank == 0 || error("expecting a scalar value for field `", key,
                               "` in OI-FITS extension ", extname)
        else
            # Check array rank.
            mult = spec.multiplier
            maxrank = (spec.type === :STRING || mult == 1 ? 1 :
                       mult == -2 ? 3 : 2)
            rank > maxrank && error("bad number of dimensions for field `",
                                    key, "` of OI-FITS extension ", extname)

            # Fields may have up to 3 dimensions.  For now, the last dimension
            # is the number of rows.  FIXME: The ordering of dimensions must
            # be changed: the number of rows should be the first dimension.
            dim0 = (rank >= 1 ? dims[end] : 1)
            dim1 = (rank >= 2 ? dims[1] : 1)
            dim2 = (rank >= 3 ? dims[2] : 1)

            if rows == -1
                rows = dim0
            else
                rows == dim0 || error("incompatible number of rows for field `",
                                      key, "` of OI-FITS extension ",
                                      extname)
            end

            if mult < 0
                # Expecting an N-by-W or N-by-W-by-W array (N is the number of
                # rows and W the number of channels).
                (mult == -2 && dim1 != dim2) &&
                    error("bad dimensions for field `", key,
                          "` of OI-FITS extension ", extname)
                if channels == -1
                    channels = dim1
                else
                    channels == dim1 || error("incompatible number of spectral",
                                              "channels for field `", key,
                                              "` of OI-FITS extension ",
                                              extname)
                end
            elseif spec.type != :STRING && dim1 != mult
                error("bad dimensions for field `", key,
                      "` of OI-FITS extension ", extname)
            end
        end
    end

    # Check that all mandatory fields have been given.
    nerrs = 0
    for key in def.fields
        spec = def.spec[key]
        if (spec.isoptional || has_member(obj, key)) == false
            warn("missing value for field `", key, "` of OI-FITS extension ",
                 extname)
            nerrs += 1
        end
    end
    nerrs == 0 || error("some mandatory fields are missing in OI-FITS ",
                        "extension ", extname)
    nothing
end

"""
    OIFITS.Builder.define(extname, revn, defs)

defines revision `revn` of the OI-FITS extension `extname`.  The format of the
OI-FITS extension is described `defs`, a vector of strings like:

    ["KEYWORD FORMAT DESCR",
      ...,
      ...,
     "---------------------------",
     "COLUMN  FORMAT DESCR",
      ...,
      ...,
      ...]

where:

- `KEYWORD` is the keyword name in FITS header.

- `COLUMN` is the column name in FITS table (`TTYPE`).

- `FORMAT` may be prefixed with a `?` to indicate an optional field and is:

  - for keywords, a single letter indicating the type,

  - for columns, a letter indicating the type followed by list of dimensions in
    parenthesis (letter `W` for a dimension means number of wavelengths).

- `DESCR` is a short description; units, if any, are indicated at the end
  between square brackets.

There may be any number of keyword definitions and any number of column
definitions, the two parts are separated by a dash line like

    "--------------".

See also [`get_definition`](@ref).

"""
function define(extname::AbstractString,
                revn::Integer,
                tbl::Vector{<:AbstractString})
    # Check OI-FITS extension name and revision number.
    (startswith(extname, "OI_") && extname == uppercase(extname) &&
     ! occursin(" ", extname)) || error("invalid OI-FITS extension name: \"",
                                        extname, "\"")
    revn ≥ 1 || error("invalid OI-FITS revision number: ", revn)
    fmtkey = format_key(extname, revn)
    haskey(FORMATS, fmtkey) && error("revision ", revn,
                                      " of OI-FITS extension ", extname,
                                      " already defined")

    # Parse table of definitions.
    function bad_definition(reason::AbstractString, extname::AbstractString,
                            revn::Integer, linenum::Integer,
                            code::AbstractString)
        error(reason, " in definition of OI-FITS extension ", extname,
              " (revision ", revn, ", line ", linenum, "): \"", code, "\"")
    end
    fields = get(FIELDS, extname, Set{Symbol}())
    def = Array{FieldDefinition}(undef, 0)
    iskeyword = true
    for rownum in 1:length(tbl)
        row = strip(tbl[rownum])
        m = match(r"^([^ ]+) +([^ ]+) +(.*)$", row)
        if m === nothing
            match(r"^-+$", row) === nothing &&
                bad_definition("syntax error", extname, revn, rownum, row)
            iskeyword = false
            continue
        end
        name = uppercase(m.captures[1])
        symb = to_fieldname(name)
        format = m.captures[2]
        descr = m.captures[3]
        isoptional = (format[1] == '?')
        i = (isoptional ? 2 : 1)
        type = get_datatype(format[i])
        type == :UNKNOWN && bad_definition("invalid type letter",
                                           extname, revn, rownum, row)
        if iskeyword
            length(format) == i ||
                bad_definition("invalid keyword format",
                               extname, revn, rownum, row)
            multiplier = 1
        else
            # Very naive code to parse the dimension list of the column format.
            (length(format) > i + 2 && format[i+1] == '('
             && format[end] == ')') ||
                 bad_definition("missing column dimension(s)",
                                extname, revn, rownum, row)
            format = uppercase(format[i+2:end-1])
            if format == "W"
                multiplier = -1
            elseif format == "W,W"
                multiplier = -2
            else
                multiplier = tryparse(Int, format)
                (multiplier === nothing || multiplier < 1) &&
                    bad_definition("invalid multiplier",
                                   extname, revn, rownum, row)
            end
        end
        mp = match(r"^(.*[^ ]) +\[([^\]])\]$", descr)
        if mp === nothing
            units = ""
        else
            descr = mp.captures[1]
            units = mp.captures[2]
        end
        push!(def, FieldDefinition(name, symb, iskeyword, isoptional,
                                   multiplier, type, units, descr))
        push!(fields, symb)
    end

    # Insert the data-block definition in the global table.
    FORMATS[fmtkey] = DataBlockDefinition(extname, revn, def)
    FIELDS[extname] = fields
    REVISIONS[extname] = max(revn, get(REVISIONS, extname, 0))
end

include("formats.jl")

end # module


text:
mutable struct BOMCPTree{B,A}
    ### for each belief node
    n_b::Vector{Int} # Number of visits, N(b)
    b_children::Vector{Vector{Int}} #Children, Ch(b)
    b_labels::Vector{B} # Labels, b
    v::Vector{Float64} # Value, V(b) (Unused)

    m_b::Vector{Int} # Number of visits from gen, B(bab')

    ### for each state-action node
    n_a::Vector{Int} #Number of times this action node has been visited
    a_children::Vector{Vector{Int}} #spnode
    p_nodes::Vector{Int} # The parent of this node (state not state ID)
    p_labels::Vector{B} # The parent of this node (state not state ID)
    a_labels::Vector{A} # The action associated with this node
    q::Vector{Float64} #Current Q estimate
    transitions::Dict{Tuple{Int64, B}, Tuple{Int, Int}} # Belief:(Bnode, n)
    r::Vector{Float64} # Mean reward

    function BOMCPTree{B,A}(sz::Int=1000) where {B,A}
        sz = min(sz, 100_000)
        return new(sizehint!(Int[], sz),
                   sizehint!(Vector{Int}[], sz),
                   sizehint!(B[], sz),
                   sizehint!(Float64[], sz),

                   sizehint!(Int[], sz),

                   sizehint!(Int[], sz),
                   sizehint!(Vector{Int}[], sz),
                   sizehint!(Int[], sz),
                   sizehint!(B[], sz),
                   sizehint!(A[], sz),
                   sizehint!(Float64[], sz),
                   Dict{Tuple{Int64, B}, Tuple{Int64,Int64,Float64}}(),
                   sizehint!(Float64[], sz)
                  )
    end
end

function insert_belief_node!(tree::BOMCPTree{B,A}, b::B) where {B,A}
    push!(tree.n_b, 0)
    push!(tree.b_children, Int[])
    push!(tree.b_labels, b)
    push!(tree.v, 0.0)

    push!(tree.m_b, 1)

    bnode = length(tree.n_b)
    return bnode
end

function insert_action_node!(tree::BOMCPTree{B,A}, bnode::Int, a::A, n0::Int, q0::Float64) where {B,A}
    push!(tree.n_a, n0)
    push!(tree.a_children, Int[])
    push!(tree.p_labels, tree.b_labels[bnode])
    push!(tree.p_nodes, bnode)
    push!(tree.a_labels, a)
    push!(tree.q, q0)
    push!(tree.r, 0.0)
    banode = length(tree.n_a)
    push!(tree.b_children[bnode], banode)
    return banode
end

Base.isempty(tree::BOMCPTree) = isempty(tree.n) && isempty(tree.q)

struct BOMCPBeliefNode{B,A}
    tree::BOMCPTree{B,A}
    index::Int
end

children(n::BOMCPBeliefNode) = n.tree.b_children[n.index]
n_children(n::BOMCPBeliefNode) = length(children(n))
isroot(n::BOMCPBeliefNode) = n.index == 1