text:
//============================================================================
// Name        :
// Author      : Avi
// Revision    : $Revision: #18 $
//
// Copyright 2009- ECMWF.
// This software is licensed under the terms of the Apache Licence version 2.0
// which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
// In applying this licence, ECMWF does not waive the privileges and immunities
// granted to it by virtue of its status as an intergovernmental organisation
// nor does it submit to any jurisdiction.
//
// Description :
//============================================================================
#include <boost/test/unit_test.hpp>

#include "ClientToServerCmd.hpp"
#include "MockServer.hpp"
#include "TestHelper.hpp"
#include "SSyncCmd.hpp"
#include "SNewsCmd.hpp"
#include "Ecf.hpp"
#include "SuiteChanged.hpp"
#include "Suite.hpp"
#include "Family.hpp"
#include "Task.hpp"
#include "Limit.hpp"

using namespace std;
using namespace ecf;

/// This test, is used to check sync with the client handles.
/// The client handles will register interest in a set of suites
/// When calling sync, check will only receive notification on our
/// our suites
// In particular when a set of suites are registered to a handle
// and we add/delete/check point/order the server passes back to the client
// the a *new* defs with all the suites corresponding to the handle
// There is a complication however, the suite are not copied/cloned
// This causes a problem, since if we create a new defs, and add
// the suites, the suite defs pointer is *NOW* corrupted, as it will point
// to the new defs. This is not an issue in real apps, since searisation
// fixes up these ptrs.

// The client handle commands do not change state & modify change number, hence need to bypass these checks
static bool bypass_state_modify_change_check = false;

BOOST_AUTO_TEST_SUITE( BaseTestSuite )

static defs_ptr create_client_defs(defs_ptr defs)
{
   for(size_t j = 0; j < 5; j++)  {
      suite_ptr suite = defs->add_suite( "s" + boost::lexical_cast<std::string>(j) );
      family_ptr f = suite->add_family("f");
      f->add_task("t");
      if (j == 0) {
         suite->addLimit( Limit("suiteLimit",10) );
         suite->addRepeat( RepeatDate("YMD",20090916,20090916,1) );
      }
   }
   return defs;
}

static defs_ptr create_server_defs()
{
   defs_ptr defs = Defs::create();

   // Create server defs, with a port other than default.
   // This allows additional testing. i.e server variables
   std::vector<Variable>  server_variables;
   ServerState::setup_default_server_variables(server_variables,"3141");
   defs->set_server().set_server_variables(server_variables);

   // ensure client/server start out the same
   return create_client_defs(defs);
}

/// define a function which returns nothing, and takes a defs_ptr parameter
typedef boost::function<bool (defs_ptr)> defs_change_cmd;

/// Re-use the same test scaffold to modify and then resync, by passing
/// in a function that will modify the defs
void test_sync_scaffold( defs_change_cmd the_defs_change_command, const std::string& test_name, bool full_sync = false)
{
   // Create the defs
   defs_ptr server_defs = create_server_defs();
   ServerReply server_reply;
   server_reply.set_client_defs( create_client_defs(Defs::create()) );

   Ecf::set_debug_equality(true); // only has affect in DEBUG build
   BOOST_CHECK_MESSAGE( *server_defs == *server_reply.client_defs(),
                        test_name << ": Starting point client and server defs should be the same : "
                        << "SERVER\n" << server_defs
                        << "CLIENT\n" << server_reply.client_defs());
   Ecf::set_debug_equality(false); // only has affect in DEBUG build

   // set handle and change numbers, before any changes
   Ecf::set_state_change_no(0);
   Ecf::set_modify_change_no(0);
   unsigned int client_state_change_no = Ecf::state_change_no();
   unsigned int client_modify_change_no = Ecf::modify_change_no();
   unsigned int client_handle = 0;

   bool expected_change;

   // make some change to the server
   {
      /// create client handle which references suites s0 and s4, in the server defs
      /// Registering suites should change handle_changed boolean
      std::vector<std::string> suite_names; suite_names.emplace_back("s0"); suite_names.emplace_back("s4");
      TestHelper::invokeRequest(server_defs.get(),Cmd_ptr( new ClientHandleCmd(0,suite_names,false)),bypass_state_modify_change_check);

      BOOST_CHECK_MESSAGE(server_defs->client_suite_mgr().clientSuites().size() == 1,test_name << ": Expected 1 Client suites but found " <<server_defs->client_suite_mgr().clientSuites().size());
      client_handle = server_defs->client_suite_mgr().clientSuites().front().handle();
      BOOST_CHECK_MESSAGE( client_handle == 1,"" );

      /// Check that handle_changed set, required for syncing, i.e needed for full sync , without change state/modify numbers
      BOOST_CHECK_MESSAGE(server_defs->client_suite_mgr().handle_changed(client_handle) == true,"Expected handle_changed to be set when suites are registered ");

      /// called create Defs should clear the flag. make sure server state, is synced
      defs_ptr the_client_defs = server_defs->client_suite_mgr().create_defs(client_handle,server_defs);
      BOOST_CHECK_MESSAGE(the_client_defs->suiteVec().size() == 2  ,test_name << ": Expected 2 suites");
      BOOST_CHECK_MESSAGE(server_defs->client_suite_mgr().handle_changed(client_handle) == false,test_name << ": Expected handle_changed to be cleared after create_defs()");
      BOOST_CHECK_MESSAGE( server_defs->server().compare(the_client_defs->server() ), test_name << ": Server state does not match");

      Ecf::set_server(true);
      expected_change = the_defs_change_command(server_defs);
      std::string error_msg;
      BOOST_REQUIRE_MESSAGE( server_defs->checkInvariants(error_msg) , test_name << ": Invariants failed: " << error_msg);
      Ecf::set_server(false);

      /// Call create defs again, after change in server defs, check server state is synced
      the_client_defs = server_defs->client_suite_mgr().create_defs(client_handle,server_defs);
      BOOST_CHECK_MESSAGE( server_defs->server().compare(the_client_defs->server() ), test_name << ": Server state does not match");
      BOOST_CHECK_MESSAGE( server_defs->get_flag() == the_client_defs->get_flag(), test_name << ": Server flags do not match");
   }

   MockServer mock_server(server_defs);
   SNewsCmd news_cmd(client_handle, client_state_change_no,  client_modify_change_no, &mock_server );
   SSyncCmd cmd(client_handle, client_state_change_no,  client_modify_change_no, &mock_server );

   if ( expected_change ) {
      BOOST_CHECK_MESSAGE( news_cmd.get_news(),         test_name << " : get_news : Expected server to change");
      BOOST_CHECK_MESSAGE( cmd.do_sync( server_reply ), test_name << " : do_sync : Expected server to change");
      BOOST_CHECK_MESSAGE( server_reply.in_sync(),      test_name << " : in_sync : Expected client server to be in sync");
      BOOST_CHECK_MESSAGE( server_reply.full_sync() == full_sync,test_name << ": Expected sync not as expected. client: " << server_reply.full_sync() << " full_sync: " << full_sync);
      BOOST_CHECK_MESSAGE( server_defs->state() == server_reply.client_defs()->state(),test_name << ": Expected server State(" << NState::toString(server_defs->state()) << ") to be same as client state(" << NState::toString(server_reply.client_defs()->state()) << ")");
      if (full_sync) {
         Ecf::set_debug_equality(true); // only has affect in DEBUG build
         BOOST_CHECK_MESSAGE( server_defs->server().compare(server_reply.client_defs()->server()),test_name << ": Server state does not match");
      }

      // * Note we expect client defs to fail invariant checking when doing a full sync with handles
      // * Under real server this should be ok since, we fix up the defs ptr, during serialisation

      // * note. We can't really compare server and client defs, since when we sync with
      // * with handles, we only return a sub set of the suites, in our handle

      // DO a sync again. hence we should expect no changes
      server_reply.clear_for_invoke(false);
      Ecf::set_server(true);
      /* server side */ SNewsCmd news_cmd1(client_handle, server_reply.client_defs()->state_change_no(),  server_reply.client_defs()->modify_change_no(), &mock_server );
      /* server side */ SSyncCmd cmd1(client_handle, server_reply.client_defs()->state_change_no(),  server_reply.client_defs()->modify_change_no(), &mock_server );
      Ecf::set_server(false);
      /* client side */ BOOST_CHECK_MESSAGE( !news_cmd1.get_news(),         test_name << ": Expected no changes to client, we should be in sync");
      /* client side */ BOOST_CHECK_MESSAGE( !cmd1.do_sync( server_reply ), test_name << ": Expected no changes to client, we should be in sync");
   }
   else {

      BOOST_CHECK_MESSAGE( !news_cmd.get_news(),         test_name << ": Expected no change");
      BOOST_CHECK_MESSAGE( !cmd.do_sync( server_reply ), test_name << ": Expected no change");
      BOOST_CHECK_MESSAGE( !server_reply.in_sync(),      test_name << ": Expected no change");
      BOOST_CHECK_MESSAGE( !(*server_defs == *server_reply.client_defs()), test_name << ": Server and client defs expected to differ" );
   }
}

static bool set_server_state_shutdown(defs_ptr defs) {
   defs->set_server().set_state( SState::SHUTDOWN );
   return true; // expect changes
}
static bool set_server_state_running(defs_ptr defs) {
   defs->set_server().set_state( SState::RUNNING );
   return true; // expect changes
}

static bool set_defs_flag(defs_ptr defs) {
   defs->flag().set( ecf::Flag::BYRULE );
   defs->flag().set( ecf::Flag::EDIT_FAILED );
   return true; // expect changes
}

static bool change_suites_s3_outside_of_handle(defs_ptr defs)
{
   /// Make a state change to suites s3, in the server. This is *not* in the client handle
   /// Note: we do *NOT* make a state change as this will be propagated to the defs, and hence will be synced
   /// We *NEED* MockSuiteChangedServer, so that change is propagated to the suite.
   MockSuiteChangedServer mockServer(defs->findSuite("s3")); // Increment suite state/modify change number
   defs->findSuite("s3")->suspend();  // small scale state change
   return false;                    // expect no changes
}

static bool change_suites_s3_outside_of_handle_add_variable(defs_ptr defs)
{
   /// make a state change to suites s3, in the server. This is *not* in the client handle
   /// We *NEED* MockSuiteChangedServer, so that change is propagated to the suite.
   MockSuiteChangedServer mockServer(defs->findSuite("s3")); // Increment suite state/modify change number
   defs->findSuite("s3")->add_variable("Var","value");       // small scale state change
   return false;                                             // expect no changes in sync
}

static bool add_task_to_suite_s3(defs_ptr defs)
{
   /// make a modify change to suites s3, in the server. This is *not* in the client handle
   MockSuiteChangedServer mockServer(defs->findSuite("s3")); // Increment suite state/modify change number
   defs->findSuite("s3")->addTask( Task::create("s3_task")); // small scale change
   return false;                                             //  expect no changes, since suite s3 not in handle
}

static bool delete_task_on_suite_s3(defs_ptr defs)
{
   /// make a modify change to suites s3, in the server. This is *not* in the client handle
   MockSuiteChangedServer mockServer(defs->findSuite("s3")); // Increment suite state/modify change number
   defs->findAbsNode("/s3/f/t")->remove(); // small scale change
   return false;                           // expect no changes, since suite s3 not in handle
}
static bool delete_family_on_suite_s3(defs_ptr defs)
{
   /// make a modify change to suites s3, in the server. This is *not* in the client handle
   MockSuiteChangedServer mockServer(defs->findSuite("s3")); // Increment suite state/modify change number
   defs->findAbsNode("/s3/f")->remove(); // small scale change
   return false;                         // expect no changes, since suite s3 not in handle
}

static bool change_state_of_s4(defs_ptr defs)
{
   /// Ok now make state change to s4, which **is** in the handle
   /// We need MockSuiteChangedServer, so that change is propagated to the suite.
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findSuite("s4")->set_state(NState::SUBMITTED);      // small scale change
   return true;                                             // expect changes
}

static bool add_variable_to_suite_s4(defs_ptr defs)
{
   /// Ok now make state change to s4, which **is** in the handle
   /// We need MockSuiteChangedServer, so that change is propagated to the suite.
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findSuite("s4")->add_variable("Var","value");       // small scale state change
   return true;                                             // expect changes
}

static bool add_server_user_variables(defs_ptr defs)
{
   // Change server. This is outside of any suites
   std::vector<Variable> user_variables;
   user_variables.emplace_back("a","b");
   user_variables.emplace_back("c","d");
   defs->set_server().set_user_variables(user_variables);
   return true; // expect change
}

static bool add_task_to_suite_s4(defs_ptr defs)
{
   /// Ok now make modify change to s4, which **is** in the handle
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findSuite("s4")->addTask( Task::create("s4_task")); // small scale change
   return true;                                             // expect changes
}

static bool delete_task_on_suite_s4(defs_ptr defs)
{
   /// Ok now make modify change to s4, which **is** in the handle
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findAbsNode("/s4/f/t")->remove();                   // small scale change
   return true;                                             // expect changes, since suite s4 is in handle
}

static bool delete_family_on_suite_s4(defs_ptr defs)
{
   /// Ok now make modify change to s4, which **is** in the handle
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findAbsNode("/s4/f")->remove();                     // small scale change
   return true;                                             // expect changes, since suite s4 is in handle
}

static bool change_order_of_s4_top(defs_ptr defs) {
   /// Ok make modify change to s4, which **is** in the handle
   suite_ptr s4 = defs->findSuite("s4");
   MockSuiteChangedServer mockServer(s4); // Increment suite state/modify change number
   defs->order(s4.get(),NOrder::TOP) ;    // small scale scale change
   return true;
}

static bool change_order_of_s4_bottom(defs_ptr defs) {
   /// Ok make modify change to s4, which **is** in the handle
   suite_ptr s4 = defs->findSuite("s4");
   MockSuiteChangedServer mockServer(s4);   // Increment suite state/modify change number
   defs->order(s4.get(),NOrder::BOTTOM) ;   // small scale change
   return true;
}

static bool delete_suite_s4(defs_ptr defs)
{
   /// Ok now make delete s4 which, which **is** in the handle
   MockSuiteChangedServer mockServer(defs->findSuite("s4")); // Increment suite state/modify change number
   defs->findSuite("s4")->remove();            // large scale change
   return true;                                // expect changes
}

static bool set_defs_state(defs_ptr defs)
{
   defs->set_state( NState::ABORTED );         // changes the defs state
   return true;                               // expect changes
}

static bool set_defs_state_2(defs_ptr defs)
{
   defs->set_state( NState::ABORTED );         // changes the defs state
   return delete_suite_s4( defs );            // large scale change
}

static bool add_server_variable(defs_ptr defs) {
   // Change defs server state. small scale change
   TestHelper::invokeRequest(defs.get(),Cmd_ptr( new AlterCmd("/",AlterCmd::ADD_VARIABLE,"_fred_","value")));
   return true;
}

static bool change_server_variable(defs_ptr defs) {
   // Because the scaffold create client/server defs each time.
   // To test change/delete variables we modify the default set
   TestHelper::invokeRequest(defs.get(),Cmd_ptr( new AlterCmd("/",AlterCmd::VARIABLE,"ECF_TRIES","4")));
   return true;
}

// ===============================================================================
// The modifiers, do this for suite s0 which is in a handle
// ===============================================================================

static bool s0_delete_some_attributes(defs_ptr defs) {
   /// Ok now make state change to s4, which **is** in the handle
   /// We need MockSuiteChangedServer, so that change is propagated to the suite.
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   std::vector<Task*> tasks;
   suite->getAllTasks(tasks);
   BOOST_REQUIRE_MESSAGE( !tasks.empty(), "Expected at least one task");

   for(Task* task: tasks) {
      SuiteChanged1 changed(task->suite());
      task->addMeter(Meter("meter",0,100));
   }
   return true;
}

static bool s0_add_some_attributes(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   std::vector<task_ptr> tasks;
   suite->get_all_tasks(tasks);
   BOOST_REQUIRE_MESSAGE( !tasks.empty(), "Expected at least one task");

   for(task_ptr task: tasks) { SuiteChanged1 changed(suite.get()); task->addDay( DayAttr(DayAttr::TUESDAY) );}
   return true;
}

static bool s0_begin(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   suite->begin();
   return true;
}

static bool s0_add_alias(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   std::vector<task_ptr> tasks;
   suite->get_all_tasks(tasks);
   BOOST_REQUIRE_MESSAGE( !tasks.empty(), "Expected at least one task");

   SuiteChanged1 changed(tasks[0]->suite());
   tasks[0]->add_alias_only();
   return true;
}

static bool s0_remove_all_tasks(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   // Remove tasks should force a incremental sync
   std::vector<task_ptr> tasks;
   suite->get_all_tasks(tasks);
   BOOST_REQUIRE_MESSAGE( !tasks.empty(), "Expected at least one task");
   for(task_ptr task: tasks) { SuiteChanged1 changed(task->suite()); task->remove() ;}

   tasks.clear();
   suite->get_all_tasks(tasks);
   BOOST_REQUIRE_MESSAGE( tasks.empty(), "Failed to delete tasks");
   return true;
}

static bool s0_remove_a_family(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   std::vector<Family*> vec;
   suite->getAllFamilies(vec);
   size_t family_size = vec.size();
   BOOST_REQUIRE_MESSAGE( !vec.empty(), "Expected at least one family");
   if (!vec.empty()) {
      SuiteChanged1 changed(vec[0]->suite());
      vec[0]->remove();
   }

   vec.clear();
   suite->getAllFamilies(vec);
   BOOST_REQUIRE_MESSAGE( vec.size() < family_size, "Failed to delete family");
   return true;
}

static bool s0_change_clock_gain(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   suite->changeClockGain("100001");
   return true;
}

static bool s0_change_clock_type_to_real(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   suite->changeClockType("hybrid");
   return true;
}

static bool s0_change_clock_date(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");

   suite->changeClockDate("1.1.2001");
   return true;
}

static bool s0_change_clock_sync(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   suite->changeClockSync();
   return true;
}

/// This has been split into two functions, as changing both together could mask an error
/// i.e found bug where we forgot to update state_change number when changing the limit
/// max value, however because we had, changed value as well it got masked.
static bool s0_change_limit_max(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   // Note: we ONLY* need MockSuiteChangedServer, when we make changes via functions and not commands

   std::vector<limit_ptr> theLimits =  suite->limits();
   BOOST_REQUIRE_MESSAGE( !theLimits.empty(),"The limit are empty on suite s0 " << defs);
   for(limit_ptr l: theLimits) {
      //std::cout << "found " << l->toString() << "\n";
      TestHelper::invokeRequest(defs.get(),Cmd_ptr( new AlterCmd(suite->absNodePath(),AlterCmd::LIMIT_MAX,l->name(),"90")));
      limit_ptr v = suite->find_limit(l->name());
      BOOST_CHECK_MESSAGE( v.get() && v->theLimit() == 90, "expected to find limit with max value of 90");
   }
   return true;
}

static bool s0_change_limit_value(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   // Note: we ONLY* need MockSuiteChangedServer, when we make changes via functions and not commands

   std::vector<limit_ptr> theLimits =  suite->limits();
   for(limit_ptr l: theLimits) {
      TestHelper::invokeRequest(defs.get(),Cmd_ptr( new AlterCmd(suite->absNodePath(),AlterCmd::LIMIT_VAL,l->name(),"33")));
      limit_ptr v = suite->find_limit(l->name());
      BOOST_CHECK_MESSAGE( v.get() && v->value() == 33, "expected to find limit with value of 33");
   }
   return true;
}

static bool s0_update_repeat(defs_ptr defs) {
   suite_ptr suite = defs->findSuite("s0");
   BOOST_REQUIRE_MESSAGE( suite,"Could not find suite");
   MockSuiteChangedServer mockServer(suite); // Increment suite state/modify change number

   SuiteChanged1 changed(suite.get());
   suite->increment_repeat();
   return true;
}

BOOST_AUTO_TEST_CASE( test_ssync_using_handle  )
{
   cout << "Base:: ...test_ssync_using_handle\n";
   TestLog test_log("test_ssync_using_handle.log"); // will create log file, and destroy log and remove file at end of scope

   // =======================================================================================
   // Note: where we update Suite::modify_change_no()  we should *EXPECT* a full sync
   // =======================================================================================

   // test_sync_scaffold will created 5 suites  s0,s1,s2,s3,s4,s5 and add suites s0 & s4 to a handle
   // The following test will perform changes in/out of handles

   {  // Change defs state in the presence of handles. These should sync regardless of handles
      // The default server state is HALTED, hence setting to halted will not show a change
      test_sync_scaffold(set_server_state_shutdown,"set_server_state_shutdown");
      test_sync_scaffold(set_server_state_running,"set_server_state_running");
      test_sync_scaffold(set_defs_flag,"set_defs_flag");

      test_sync_scaffold(add_server_variable,"add_server_variable");
      test_sync_scaffold(change_server_variable,"change_server_variable");

      test_sync_scaffold(set_defs_state,"set_defs_state");
      test_sync_scaffold(set_defs_state_2,"set_defs_state_2",true /* expect a full sync */);
   }

   test_sync_scaffold(change_suites_s3_outside_of_handle,"change_suites_s3_outside_of_handle");
   test_sync_scaffold(change_suites_s3_outside_of_handle_add_variable,"change_suites_s3_outside_of_handle_add_variable");
   test_sync_scaffold(add_task_to_suite_s3,"add_task_to_suite_s3");
   test_sync_scaffold(delete_task_on_suite_s3,"delete_task_on_suite_s3");
   test_sync_scaffold(delete_family_on_suite_s3,"delete_family_on_suite_s3");

   test_sync_scaffold(change_order_of_s4_top,"change_order_of_s4_top");    // change order is an incremental sync
   test_sync_scaffold(change_order_of_s4_bottom,"change_order_of_s4_bottom"); // change order is an incremental sync

   test_sync_scaffold(change_state_of_s4,"change_state_of_s4");
   test_sync_scaffold(add_variable_to_suite_s4,"add_variable_to_suite_s4");
   test_sync_scaffold(add_server_user_variables,"add_server_user_variables");
   test_sync_scaffold(add_task_to_suite_s4,"add_task_to_suite_s4");
   test_sync_scaffold(delete_task_on_suite_s4,"delete_task_on_suite_s4");
   test_sync_scaffold(delete_family_on_suite_s4,"delete_family_on_suite_s4");
   test_sync_scaffold(delete_suite_s4,"delete_suite_s4", true /* expect a full sync */);

   test_sync_scaffold(s0_delete_some_attributes,"s0_delete_some_attributes");
   test_sync_scaffold(s0_add_some_attributes,"s0_add_some_attributes");
   test_sync_scaffold(s0_add_alias,"s0_add_alias");
   test_sync_scaffold(s0_update_repeat,"s0_update_repeat");
   test_sync_scaffold(s0_change_limit_max,"s0_change_limit_max");
   test_sync_scaffold(s0_change_limit_value,"s0_change_limit_value");
   test_sync_scaffold(s0_begin,"s0_begin", true/* expect a full sync */);

   test_sync_scaffold(s0_remove_all_tasks,"s0_remove_all_tasks" );
   test_sync_scaffold(s0_remove_a_family,"s0_remove_a_family");

   test_sync_scaffold(s0_change_clock_gain,"s0_change_clock_gain", true/* expect a full sync */);
   test_sync_scaffold(s0_change_clock_type_to_real,"s0_change_clock_type_to_real", true/* expect a full sync */);
   test_sync_scaffold(s0_change_clock_date,"s0_change_clock_date", true/* expect a full sync */);
   test_sync_scaffold(s0_change_clock_sync,"s0_change_clock_sync", true/* expect a full sync */);
}

// =============================================================================================

static defs_ptr create_the_server_defs()
{
   defs_ptr defs = create_server_defs();
   std::vector<suite_ptr> suite_vec =  defs->suiteVec();
   for(size_t j = 0; j < suite_vec.size(); j++)  {
      suite_vec[j]->set_state_change_no(j);
      suite_vec[j]->set_modify_change_no(j);
   }
   return defs;
}

BOOST_AUTO_TEST_CASE( test_ssync_full_sync_using_handle  )
{
   /// This test checks that when user has registered with all the suites.
   /// Syncing should use the global change numbers
   /// **ADDITIONALLY* the newsCmd  must reflect this.
   /// This is handled in ClientSuites::create_defs, in that we *MUST* update the
   /// local change numbers to be the same as the global change numbers
   /// This is important since the NewsCmd must be in *SYNC* with SYNCCmd

   cout << "Base:: ...test_ssync_full_sync_using_handle\n";
   TestLog test_log("test_ssync_full_sync_using_handle.log"); // will create log file, and destroy log and remove file at end of scope

   // Create the server defs with some changes, in state & modify numbers
   defs_ptr server_defs = create_the_server_defs();

   // Create Client defs, without any changes
   ServerReply server_reply;
   server_reply.set_client_defs( create_client_defs(Defs::create()) );

   // Server & client should be the same, since we ignore change numbers in the comparison
   DebugEquality debug_equality; // only as affect in DEBUG build
   BOOST_CHECK_MESSAGE( *server_defs == *server_reply.client_defs(), "Starting point client and server defs should be the same"
                        << "SERVER\n" << server_defs
                        << "CLIENT\n" << server_reply.client_defs());

   /// register interest in **ALL** the suites
   std::vector<std::string> suite_names;
   suite_names.emplace_back("s0");
   suite_names.emplace_back("s1");
   suite_names.emplace_back("s2");
   suite_names.emplace_back("s3");
   suite_names.emplace_back("s4");
   TestHelper::invokeRequest(server_defs.get(),Cmd_ptr( new ClientHandleCmd(0,suite_names,false)),bypass_state_modify_change_check);

   /// make sure handle created.
   BOOST_CHECK_MESSAGE(server_defs->client_suite_mgr().clientSuites().size() == 1,"Expected 1 Client suites but found " <<server_defs->client_suite_mgr().clientSuites().size());
   unsigned int client_handle = server_defs->client_suite_mgr().clientSuites().front().handle();
   BOOST_CHECK_MESSAGE( client_handle == 1,"" );

   // make sure server suites have different state/modify numbers from the client
   {
      unsigned int server_state_change_no = 0;
      unsigned int server_modify_change_no = 0;
      server_defs->client_suite_mgr().max_change_no(client_handle,server_state_change_no, server_modify_change_no);
      BOOST_CHECK_MESSAGE(server_state_change_no == 4,"" );
      BOOST_CHECK_MESSAGE(server_modify_change_no == 4,"" );

      // *MAKE* sure global change numbers are different to handle suite change numbers
      // This is the key part of this test.
      // Since syncing should transfer these change numbers to the client
      Ecf::set_state_change_no(server_state_change_no+10);
      Ecf::set_modify_change_no(server_modify_change_no+20);
   }

   // Now sync from server
   Ecf::set_server(true);
   MockServer mock_server(server_defs);
   /* server side */ SNewsCmd news_cmd(client_handle, server_reply.client_defs()->state_change_no(), server_reply.client_defs()->modify_change_no(), &mock_server );
   /* server side */ SSyncCmd cmd(client_handle, server_reply.client_defs()->state_change_no(), server_reply.client_defs()->modify_change_no(), &mock_server );
   Ecf::set_server(false);

   // make sure SSyncCmd updated the server change numbers, to use global change numbers
   unsigned int server_state_change_no = 0;
   unsigned int server_modify_change_no = 0;
   {
      server_defs->client_suite_mgr().max_change_no(client_handle,server_state_change_no, server_modify_change_no);
      BOOST_CHECK_MESSAGE(server_state_change_no == 14,"" );
      BOOST_CHECK_MESSAGE(server_modify_change_no == 24,"" );
   }

   // SYNC the client, via do_sync( this should transfer change numbers to the client )
   BOOST_CHECK_MESSAGE( news_cmd.get_news(), "Expected server to change");
   BOOST_CHECK_MESSAGE( cmd.do_sync( server_reply ), "Expected server to change");
   BOOST_CHECK_MESSAGE( server_reply.in_sync(),      "Expected server to change");
   BOOST_CHECK_MESSAGE( server_defs->state() == server_reply.client_defs()->state(),"Expected server State(" << NState::toString(server_defs->state()) << ") to be same as client state(" << NState::toString(server_reply.client_defs()->state()) << ")");

   // After do_sync client and server change number should be in sync
   BOOST_CHECK_MESSAGE(server_reply.client_defs()->state_change_no() == server_state_change_no,"Expected " << server_reply.client_defs()->state_change_no() << " state change number but found " << server_state_change_no);
   BOOST_CHECK_MESSAGE(server_reply.client_defs()->modify_change_no() == server_modify_change_no,"Expected " << server_reply.client_defs()->modify_change_no() << " modify change number but found " << server_modify_change_no);

   // Do final sync, there should not be ANY changes
   Ecf::set_server(true);
   /* server side */ SNewsCmd news_cmd1(client_handle, server_reply.client_defs()->state_change_no(),  server_reply.client_defs()->modify_change_no(), &mock_server );
   /* server side */ SSyncCmd cmd1(client_handle, server_reply.client_defs()->state_change_no(),  server_reply.client_defs()->modify_change_no(), &mock_server );
   Ecf::set_server(false);
   /* client side */ BOOST_CHECK_MESSAGE( !news_cmd1.get_news(),         "Expected no changes to client, we should be in sync");
   /* client side */ BOOST_CHECK_MESSAGE( !cmd1.do_sync( server_reply ), "Expected no changes to client, we should be in sync");
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: mpartio/ecflow

max_issues_repo_name: mpartio/ecflow

max_forks_repo_name: mpartio/ecflow


text:
/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2017 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

// mapnik
#include <mapnik/debug.hpp>
#include <mapnik/image_reader.hpp>
#include <mapnik/util/char_array_buffer.hpp>
extern "C"
{
#include <tiffio.h>
}

#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/streams/bufferstream.hpp>
#pragma GCC diagnostic pop
#include <mapnik/mapped_memory_cache.hpp>
#endif

// stl
#include <memory>
#include <fstream>

namespace mapnik { namespace detail {

static toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence)
{
    std::istream* in = reinterpret_cast<std::istream*>(handle);

    switch(whence)
    {
    case SEEK_SET:
        in->seekg(off, std::ios_base::beg);
        break;
    case SEEK_CUR:
        in->seekg(off, std::ios_base::cur);
        break;
    case SEEK_END:
        in->seekg(off, std::ios_base::end);
        break;
    }
    return static_cast<toff_t>(in->tellg());
}

static int tiff_close_proc(thandle_t)
{
    return 0;
}

static toff_t tiff_size_proc(thandle_t handle)
{
    std::istream* in = reinterpret_cast<std::istream*>(handle);
    std::ios::pos_type pos = in->tellg();
    in->seekg(0, std::ios::end);
    std::ios::pos_type len = in->tellg();
    in->seekg(pos);
    return static_cast<toff_t>(len);
}

static tsize_t tiff_read_proc(thandle_t handle, tdata_t buf, tsize_t size)
{
    std::istream * in = reinterpret_cast<std::istream*>(handle);
    std::streamsize request_size = size;
    if (static_cast<tsize_t>(request_size) != size)
        return static_cast<tsize_t>(-1);
    in->read(reinterpret_cast<char*>(buf), request_size);
    return static_cast<tsize_t>(in->gcount());
}

static tsize_t tiff_write_proc(thandle_t , tdata_t , tsize_t)
{
    return 0;
}

static void tiff_unmap_proc(thandle_t, tdata_t, toff_t)
{
}

static int tiff_map_proc(thandle_t, tdata_t* , toff_t*)
{
    return 0;
}

template <typename T>
struct tiff_io_traits
{
    using input_stream_type = std::istream;
};

#if defined(MAPNIK_MEMORY_MAPPED_FILE)
template <>
struct tiff_io_traits<boost::interprocess::ibufferstream>
{
    using input_stream_type = boost::interprocess::ibufferstream;
};
#endif
}

template <typename T>
class tiff_reader : public image_reader
{
    using tiff_ptr = std::shared_ptr<TIFF>;
    using source_type = T;
    using input_stream = typename detail::tiff_io_traits<source_type>::input_stream_type;
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
    mapnik::mapped_region_ptr mapped_region_;
#endif

    struct tiff_closer
    {
        void operator() (TIFF * tif)
        {
            if (tif != 0) TIFFClose(tif);
        }
    };

private:
    source_type source_;
    input_stream stream_;
    tiff_ptr tif_;
    int read_method_;
    int rows_per_strip_;
    int tile_width_;
    int tile_height_;
    std::size_t width_;
    std::size_t height_;
    boost::optional<box2d<double> > bbox_;
    unsigned bps_;
    unsigned sample_format_;
    unsigned photometric_;
    unsigned bands_;
    unsigned planar_config_;
    unsigned compression_;
    bool has_alpha_;
    bool is_tiled_;

public:
    enum TiffType {
        generic=1,
        stripped,
        tiled
    };
    explicit tiff_reader(std::string const& filename);
    tiff_reader(char const* data, std::size_t size);
    virtual ~tiff_reader();
    unsigned width() const final;
    unsigned height() const final;
    boost::optional<box2d<double> > bounding_box() const final;
    inline bool has_alpha() const final { return has_alpha_; }
    void read(unsigned x,unsigned y,image_rgba8& image) final;
    image_any read(unsigned x, unsigned y, unsigned width, unsigned height) final;
    // methods specific to tiff reader
    unsigned bits_per_sample() const { return bps_; }
    unsigned sample_format() const { return sample_format_; }
    unsigned photometric() const { return photometric_; }
    bool is_tiled() const { return is_tiled_; }
    unsigned tile_width() const { return tile_width_; }
    unsigned tile_height() const { return tile_height_; }
    unsigned rows_per_strip() const { return rows_per_strip_; }
    unsigned planar_config() const { return planar_config_; }
    unsigned compression() const { return compression_; }
private:
    tiff_reader(const tiff_reader&);
    tiff_reader& operator=(const tiff_reader&);
    void init();

    template <typename ImageData>
    void read_generic(std::size_t x,std::size_t y, ImageData & image);

    template <typename ImageData>
    void read_stripped(std::size_t x,std::size_t y, ImageData & image);

    template <typename ImageData>
    void read_tiled(std::size_t x,std::size_t y, ImageData & image);

    template <typename ImageData>
    image_any read_any_gray(std::size_t x, std::size_t y, std::size_t width, std::size_t height);

    TIFF* open(std::istream & input);
};

namespace
{

image_reader* create_tiff_reader(std::string const& filename)
{
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
    return new tiff_reader<boost::interprocess::ibufferstream>(filename);
#else
    return new tiff_reader<std::filebuf>(filename);
#endif
}

image_reader* create_tiff_reader2(char const * data, std::size_t size)
{
    return new tiff_reader<mapnik::util::char_array_buffer>(data, size);
}

const bool registered = register_image_reader("tiff",create_tiff_reader);
const bool registered2 = register_image_reader("tiff", create_tiff_reader2);

}

template <typename T>
tiff_reader<T>::tiff_reader(std::string const& filename)
    :
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
    stream_(),
#else
    source_(),
    stream_(&source_),
#endif

    tif_(nullptr),
    read_method_(generic),
    rows_per_strip_(0),
    tile_width_(0),
    tile_height_(0),
    width_(0),
    height_(0),
    bps_(0),
    sample_format_(SAMPLEFORMAT_UINT),
    photometric_(0),
    bands_(1),
    planar_config_(PLANARCONFIG_CONTIG),
    compression_(COMPRESSION_NONE),
    has_alpha_(false),
    is_tiled_(false)
{

#if defined(MAPNIK_MEMORY_MAPPED_FILE)
     boost::optional<mapnik::mapped_region_ptr> memory =
         mapnik::mapped_memory_cache::instance().find(filename,true);

     if (memory)
     {
         mapped_region_ = *memory;
         stream_.buffer(static_cast<char*>(mapped_region_->get_address()),mapped_region_->get_size());
     }
     else
     {
         throw image_reader_exception("could not create file mapping for " + filename);
     }
#else
     source_.open(filename, std::ios_base::in | std::ios_base::binary);
#endif
     if (!stream_) throw image_reader_exception("TIFF reader: cannot open file " + filename);
     init();
}

template <typename T>
tiff_reader<T>::tiff_reader(char const* data, std::size_t size)
    : source_(data, size),
      stream_(&source_),
      tif_(nullptr),
      read_method_(generic),
      rows_per_strip_(0),
      tile_width_(0),
      tile_height_(0),
      width_(0),
      height_(0),
      bps_(0),
      sample_format_(SAMPLEFORMAT_UINT),
      photometric_(0),
      bands_(1),
      planar_config_(PLANARCONFIG_CONTIG),
      compression_(COMPRESSION_NONE),
      has_alpha_(false),
      is_tiled_(false)
{
    if (!stream_) throw image_reader_exception("TIFF reader: cannot open image stream ");
    init();
}

template <typename T>
void tiff_reader<T>::init()
{
    // avoid calling TIFFs global structures
    TIFFSetWarningHandler(0);
    TIFFSetErrorHandler(0);

    TIFF* tif = open(stream_);

    if (!tif) throw image_reader_exception("Can't open tiff file");

    TIFFGetField(tif,TIFFTAG_BITSPERSAMPLE,&bps_);
    TIFFGetField(tif,TIFFTAG_SAMPLEFORMAT,&sample_format_);
    TIFFGetField(tif,TIFFTAG_PHOTOMETRIC,&photometric_);
    TIFFGetField(tif, TIFFTAG_SAMPLESPERPIXEL, &bands_);

    MAPNIK_LOG_DEBUG(tiff_reader) << "bits per sample: " << bps_ ;
    MAPNIK_LOG_DEBUG(tiff_reader) << "sample format: " << sample_format_ ;
    MAPNIK_LOG_DEBUG(tiff_reader) << "photometric: " << photometric_ ;
    MAPNIK_LOG_DEBUG(tiff_reader) << "bands: " << bands_ ;

    TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &width_);
    TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &height_);

    TIFFGetField(tif, TIFFTAG_PLANARCONFIG, &planar_config_);
    TIFFGetField(tif, TIFFTAG_COMPRESSION, &compression_ );

    std::uint16_t orientation;
    if (TIFFGetField(tif, TIFFTAG_ORIENTATION, &orientation) == 0)
    {
        orientation = 1;
    }
    MAPNIK_LOG_DEBUG(tiff_reader) << "orientation: " << orientation ;
    MAPNIK_LOG_DEBUG(tiff_reader) << "planar-config: " << planar_config_ ;
    is_tiled_ = TIFFIsTiled(tif);

    if (is_tiled_)
    {
        TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tile_width_);
        TIFFGetField(tif, TIFFTAG_TILELENGTH, &tile_height_);
        MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is tiled";
        read_method_ = tiled;
    }
    else if (TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &rows_per_strip_) != 0)
    {
        MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is stripped";
        read_method_ = stripped;
    }
    //TIFFTAG_EXTRASAMPLES
    uint16 extrasamples = 0;
    uint16* sampleinfo = nullptr;
    if (TIFFGetField(tif, TIFFTAG_EXTRASAMPLES,
                     &extrasamples, &sampleinfo))
    {
        has_alpha_ = true;
        if (extrasamples > 0 &&
            sampleinfo[0] == EXTRASAMPLE_UNSPECIFIED)
        {
            throw image_reader_exception("Unspecified provided for extra samples to tiff reader.");
        }
    }
    // Try extracting bounding box from geoTIFF tags
    {
        uint16 count = 0;
        double *pixelscale;
        double *tilepoint;
        if (TIFFGetField(tif, 33550, &count, &pixelscale) == 1 && count == 3
            && TIFFGetField(tif, 33922 , &count,  &tilepoint) == 1 && count == 6)
        {
            MAPNIK_LOG_DEBUG(tiff_reader) << "PixelScale:" << pixelscale[0] << "," << pixelscale[1] << "," <<  pixelscale[2] ;
            MAPNIK_LOG_DEBUG(tiff_reader) << "TilePoint:" << tilepoint[0] << "," << tilepoint[1] << "," << tilepoint[2] ;
            MAPNIK_LOG_DEBUG(tiff_reader) << "          " << tilepoint[3] << "," << tilepoint[4] << "," << tilepoint[5] ;

            // assuming upper-left
            double lox = tilepoint[3];
            double loy = tilepoint[4];
            double hix = lox + pixelscale[0] * width_;
            double hiy = loy - pixelscale[1] * height_;
            bbox_.reset(box2d<double>(lox, loy, hix, hiy));
            MAPNIK_LOG_DEBUG(tiff_reader) << "Bounding Box:" << *bbox_ ;
        }

    }
    if (!is_tiled_ &&
        compression_ == COMPRESSION_NONE &&
        planar_config_ == PLANARCONFIG_CONTIG)
    {
        if (height_ > 128 * 1024 * 1024)
        {
            std::size_t line_size = (bands_ * width_ * bps_ + 7) / 8;
            std::size_t default_strip_height = 8192 / line_size;
            if (default_strip_height == 0) default_strip_height = 1;
            std::size_t num_strips = height_ / default_strip_height;
            if (num_strips > 128 * 1024 * 1024)
            {
                throw image_reader_exception("Can't allocate tiff");
            }
        }
    }
}

template <typename T>
tiff_reader<T>::~tiff_reader()
{
}

template <typename T>
unsigned tiff_reader<T>::width() const
{
    return width_;
}

template <typename T>
unsigned tiff_reader<T>::height() const
{
    return height_;
}

template <typename T>
boost::optional<box2d<double> > tiff_reader<T>::bounding_box() const
{
    return bbox_;
}

template <typename T>
void tiff_reader<T>::read(unsigned x,unsigned y,image_rgba8& image)
{
    if (read_method_==stripped)
    {
        read_stripped(static_cast<std::size_t>(x),static_cast<std::size_t>(y),image);
    }
    else if (read_method_==tiled)
    {
        read_tiled(static_cast<std::size_t>(x),static_cast<std::size_t>(y),image);
    }
    else
    {
        read_generic(static_cast<std::size_t>(x),static_cast<std::size_t>(y),image);
    }
}

template <typename T>
template <typename ImageData>
image_any tiff_reader<T>::read_any_gray(std::size_t x0, std::size_t y0, std::size_t width, std::size_t height)
{
    using image_type = ImageData;
    using pixel_type = typename image_type::pixel_type;
    if (read_method_ == tiled)
    {
        image_type data(width, height);
        read_tiled<image_type>(x0, y0, data);
        return image_any(std::move(data));
    }
    else if (read_method_ == stripped)
    {
        image_type data(width, height);
        read_stripped<image_type>(x0, y0, data);
        return image_any(std::move(data));
    }
    else
    {
        TIFF* tif = open(stream_);
        if (tif)
        {
            image_type data(width, height);
            std::size_t block_size = rows_per_strip_ > 0 ? rows_per_strip_ : tile_height_ ;
            std::size_t start_y = y0 - y0 % block_size;
            std::size_t end_y = std::min(y0 + height, height_);
            std::size_t start_x = x0;
            std::size_t end_x = std::min(x0 + width, width_);
            std::size_t element_size = sizeof(pixel_type);
            MAPNIK_LOG_DEBUG(tiff_reader) << "SCANLINE SIZE=" << TIFFScanlineSize(tif);
            std::size_t size_to_allocate = (TIFFScanlineSize(tif) + element_size - 1)/element_size;
            std::unique_ptr<pixel_type[]> const scanline(new pixel_type[size_to_allocate]);
            if (planar_config_ == PLANARCONFIG_CONTIG)
            {
                for  (std::size_t y = start_y; y < end_y; ++y)
                {
                    // we have to read all scanlines sequentially from start_y
                    // to be able to use scanline interface with compressed blocks.
                    if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0))
                    {
                        pixel_type * row = data.get_row(y - y0);
                        if (bands_ == 1)
                        {
                            std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;});
                        }
                        else if (size_to_allocate == bands_ * width_)
                        {
                            // bands_ > 1 => packed bands in grayscale image e.g an extra alpha channel.
                            // Just pick first one for now.
                            pixel_type * buf = scanline.get() + start_x * bands_;
                            std::size_t x_index = 0;
                            for (std::size_t j = 0; j < end_x * bands_; ++j)
                            {
                                if (x_index >= width) break;
                                if (j % bands_ == 0)
                                {
                                    row[x_index++] = buf[j];
                                }
                            }
                        }
                    }
                }
            }
            else if (planar_config_ == PLANARCONFIG_SEPARATE)
            {
                for (std::size_t s = 0 ; s < bands_ ; ++s)
                {
                    for  (std::size_t y = start_y; y < end_y; ++y)
                    {
                        if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0))
                        {
                            pixel_type * row = data.get_row(y - y0);
                            std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;});
                        }
                    }
                }
            }
            return image_any(std::move(data));
        }
    }
    return image_any();
}

namespace detail {

struct rgb8
{
    std::uint8_t r;
    std::uint8_t g;
    std::uint8_t b;
};

struct rgb8_to_rgba8
{
    std::uint32_t operator() (rgb8 const& in) const
    {
        return ((255 << 24) | (in.r) | (in.g << 8) | (in.b << 16));
    }
};

template <typename T>
struct tiff_reader_traits
{
    using image_type = T;
    using pixel_type = typename image_type::pixel_type;

    constexpr static bool reverse = false;
    static bool read_tile(TIFF * tif, std::size_t x, std::size_t y, pixel_type* buf, std::size_t tile_width, std::size_t tile_height)
    {
        std::uint32_t tile_size = TIFFTileSize(tif);
        return (TIFFReadEncodedTile(tif, TIFFComputeTile(tif, x, y, 0, 0), buf, tile_size) != -1);
    }

    static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf)
    {
        return (TIFFReadEncodedStrip(tif, y/rows_per_strip, buf, -1) != -1);
    }
};

// default specialization that expands into RGBA
template <>
struct tiff_reader_traits<image_rgba8>
{
    using image_type = image_rgba8;
    using pixel_type = std::uint32_t;
    constexpr static bool reverse = true;
    static bool read_tile(TIFF * tif, std::size_t x0, std::size_t y0, pixel_type* buf, std::size_t tile_width, std::size_t tile_height)
    {
        return (TIFFReadRGBATile(tif, x0, y0, buf) != 0);
    }

    static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf)
    {
        return (TIFFReadRGBAStrip(tif, y, buf) != 0);
    }
};

}

template <typename T>
image_any tiff_reader<T>::read(unsigned x, unsigned y, unsigned width, unsigned height)
{
    if (width > 10000 || height > 10000)
    {
        throw image_reader_exception("Can't allocate tiff > 10000x10000");
    }
    std::size_t x0 = static_cast<std::size_t>(x);
    std::size_t y0 = static_cast<std::size_t>(y);
    switch (photometric_)
    {
    case PHOTOMETRIC_MINISBLACK:
    case PHOTOMETRIC_MINISWHITE:
    {
        switch (bps_)
        {
        case 8:
        {
            switch (sample_format_)
            {
            case SAMPLEFORMAT_UINT:
            {
                return read_any_gray<image_gray8>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_INT:
            {
                return read_any_gray<image_gray8s>(x0, y0, width, height);
            }
            default:
            {
                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
            }
            }
        }
        case 16:
        {
            switch (sample_format_)
            {
            case SAMPLEFORMAT_UINT:
            {
                return read_any_gray<image_gray16>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_INT:
            {
                return read_any_gray<image_gray16s>(x0, y0, width, height);
            }
            default:
            {
                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
            }
            }
        }
        case 32:
        {
            switch (sample_format_)
            {
            case SAMPLEFORMAT_UINT:
            {
                return read_any_gray<image_gray32>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_INT:
            {
                return read_any_gray<image_gray32s>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_IEEEFP:
            {
                return read_any_gray<image_gray32f>(x0, y0, width, height);
            }
            default:
            {
                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
            }
            }
        }
        case 64:
        {
            switch (sample_format_)
            {
            case SAMPLEFORMAT_UINT:
            {
                return read_any_gray<image_gray64>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_INT:
            {
                return read_any_gray<image_gray64s>(x0, y0, width, height);
            }
            case SAMPLEFORMAT_IEEEFP:
            {
                return read_any_gray<image_gray64f>(x0, y0, width, height);
            }
            default:
            {
                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
            }
            }
        }
        }
    }
    default:
    {
        //PHOTOMETRIC_PALETTE = 3;
        //PHOTOMETRIC_MASK = 4;
        //PHOTOMETRIC_SEPARATED = 5;
        //PHOTOMETRIC_YCBCR = 6;
        //PHOTOMETRIC_CIELAB = 8;
        //PHOTOMETRIC_ICCLAB = 9;
        //PHOTOMETRIC_ITULAB = 10;
        //PHOTOMETRIC_LOGL = 32844;
        //PHOTOMETRIC_LOGLUV = 32845;
        image_rgba8 data(width,height, true, true);
        read(x0, y0, data);
        return image_any(std::move(data));
    }
    }
    return image_any();
}

template <typename T>
template <typename ImageData>
void tiff_reader<T>::read_generic(std::size_t, std::size_t, ImageData &)
{
    throw image_reader_exception("tiff_reader: TODO - tiff is not stripped or tiled");
}

template <typename T>
template <typename ImageData>
void tiff_reader<T>::read_tiled(std::size_t x0,std::size_t y0, ImageData & image)
{
    using pixel_type = typename detail::tiff_reader_traits<ImageData>::pixel_type;

    TIFF* tif = open(stream_);
    if (tif)
    {
        std::uint32_t tile_size = TIFFTileSize(tif);
        std::unique_ptr<pixel_type[]> tile(new pixel_type[tile_size]);
        std::size_t width = image.width();
        std::size_t height = image.height();
        std::size_t start_y = (y0 / tile_height_) * tile_height_;
        std::size_t end_y = ((y0 + height) / tile_height_ + 1) * tile_height_;
        std::size_t start_x = (x0 / tile_width_) * tile_width_;
        std::size_t end_x = ((x0 + width) / tile_width_ + 1) * tile_width_;
        end_y = std::min(end_y, height_);
        end_x = std::min(end_x, width_);
        bool pick_first_band = (bands_ > 1) && (tile_size / (tile_width_ * tile_height_ * sizeof(pixel_type)) == bands_);
        for (std::size_t y = start_y; y < end_y; y += tile_height_)
        {
            std::size_t ty0 = std::max(y0, y) - y;
            std::size_t ty1 = std::min(height + y0, y + tile_height_) - y;

            for (std::size_t x = start_x; x < end_x; x += tile_width_)
            {
                if (!detail::tiff_reader_traits<ImageData>::read_tile(tif, x, y, tile.get(), tile_width_, tile_height_))
                {
                    MAPNIK_LOG_DEBUG(tiff_reader) <<  "read_tile(...) failed at " << x << "/" << y << " for " << width_ << "/" << height_ << "\n";
                    break;
                }
                if (pick_first_band)
                {
                    std::uint32_t size = tile_width_ * tile_height_ * sizeof(pixel_type);
                    for (std::uint32_t n = 0; n < size; ++n)
                    {
                        tile[n] = tile[n * bands_];
                    }
                }
                std::size_t tx0 = std::max(x0, x);
                std::size_t tx1 = std::min(width + x0, x + tile_width_);
                std::size_t row_index = y + ty0 - y0;

                if (detail::tiff_reader_traits<ImageData>::reverse)
                {
                    for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index)
                    {
                        // This is in reverse because the TIFFReadRGBATile reads are inverted
                        image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[(tile_height_ - ty - 1) * tile_width_ + tx0 - x]);
                    }
                }
                else
                {
                    for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index)
                    {
                        image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[ty * tile_width_ + tx0 - x]);
                    }
                }
            }
        }
    }
}

template <typename T>
template <typename ImageData>
void tiff_reader<T>::read_stripped(std::size_t x0, std::size_t y0, ImageData & image)
{
    using pixel_type = typename detail::tiff_reader_traits<ImageData>::pixel_type;
    TIFF* tif = open(stream_);
    if (tif)
    {
        std::uint32_t strip_size = TIFFStripSize(tif);
        std::unique_ptr<pixel_type[]> strip(new pixel_type[strip_size]);
        std::size_t width = image.width();
        std::size_t height = image.height();

        std::size_t start_y = (y0 / rows_per_strip_) * rows_per_strip_;
        std::size_t end_y = std::min(y0 + height, height_);
        std::size_t tx0, tx1, ty0, ty1;
        tx0 = x0;
        tx1 = std::min(width + x0, width_);
        std::size_t row = 0;
        bool pick_first_band = (bands_ > 1) && (strip_size / (width_ * rows_per_strip_ * sizeof(pixel_type)) == bands_);
        for (std::size_t y = start_y; y < end_y; y += rows_per_strip_)
        {
            ty0 = std::max(y0, y) - y;
            ty1 = std::min(end_y, y + rows_per_strip_) - y;

            if (!detail::tiff_reader_traits<ImageData>::read_strip(tif, y, rows_per_strip_, width_, strip.get()))
            {
                MAPNIK_LOG_DEBUG(tiff_reader) << "TIFFRead(Encoded|RGBA)Strip failed at " << y << " for " << width_ << "/" << height_ << "\n";
                break;
            }
            if (pick_first_band)
            {
                std::uint32_t size = width_ * rows_per_strip_ * sizeof(pixel_type);
                for (std::uint32_t n = 0; n < size; ++n)
                {
                    strip[n] = strip[bands_ * n];
                }
            }

            if (detail::tiff_reader_traits<ImageData>::reverse)
            {
                std::size_t num_rows = std::min(height_ - y, static_cast<std::size_t>(rows_per_strip_));
                for (std::size_t ty = ty0; ty < ty1; ++ty)
                {
                    // This is in reverse because the TIFFReadRGBAStrip reads are inverted
                    image.set_row(row++, tx0 - x0, tx1 - x0, &strip[(num_rows - ty - 1) * width_ + tx0]);
                }
            }
            else
            {
                for (std::size_t ty = ty0; ty < ty1; ++ty)
                {
                    image.set_row(row++, tx0 - x0, tx1 - x0, &strip[ty * width_ + tx0]);
                }
            }
        }
    }
}

template <typename T>
TIFF* tiff_reader<T>::open(std::istream & input)
{
    if (!tif_)
    {
        tif_ = tiff_ptr(TIFFClientOpen("tiff_input_stream", "rcm",
                                       reinterpret_cast<thandle_t>(&input),
                                       detail::tiff_read_proc,
                                       detail::tiff_write_proc,
                                       detail::tiff_seek_proc,
                                       detail::tiff_close_proc,
                                       detail::tiff_size_proc,
                                       detail::tiff_map_proc,
                                       detail::tiff_unmap_proc), tiff_closer());
    }
    return tif_.get();
}

} // namespace mapnik

ext: cpp


text:
//---------------------------------------------------------- -*- Mode: C++ -*-
// $Id: cptokfs_main.cc 108 2008-07-30 06:23:39Z sriramsrao $
//
// Created 2006/06/23
// Author: Sriram Rao
//
// Copyright 2008 Quantcast Corp.
// Copyright 2006-2008 Kosmix Corp.
//
// This file is part of Kosmos File System (KFS).
//
// Licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// \brief Tool that copies a file/directory from a local file system to
// KFS.  This tool is analogous to dump---backup a directory from a
// file system into KFS.
//
//----------------------------------------------------------------------------

#include <iostream>
#include <fstream>
#include <cerrno>
#include <boost/scoped_array.hpp>
using boost::scoped_array;
using std::ios_base;

extern "C" {
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <openssl/md5.h>
}

#include "libkfsClient/KfsClient.h"
#include "common/log.h"

#define MAX_FILE_NAME_LEN 256

using std::cout;
using std::endl;
using std::ifstream;
using std::ofstream;
using namespace KFS;

KfsClientPtr gKfsClient;
bool doMkdirs(const char *path);

//
// For the purpose of the cp -r, take the leaf from sourcePath and
// make that directory in kfsPath.
//
void MakeKfsLeafDir(const string &sourcePath, string &kfsPath);

//
// Given a file defined by sourcePath, copy it to KFS as defined by
// kfsPath
//
int BackupFile(const string &sourcePath, const string &kfsPath);

// Given a dirname, backit up it to dirname.  Dirname will be created
// if it doesn't exist.
void BackupDir(const string &dirname, string &kfsdirname);

// Guts of the work
int BackupFile2(string srcfilename, string kfsfilename);

int
main(int argc, char **argv)
{
    DIR *dirp;
    string kfsPath = "";
    string serverHost = "";
    int port = -1;
    char *sourcePath = NULL;
    bool help = false;
    bool verboseLogging = false;
    char optchar;
    struct stat statInfo;

    KFS::MsgLogger::Init(NULL);

    while ((optchar = getopt(argc, argv, "d:hk:p:s:v")) != -1) {
        switch (optchar) {
            case 'd':
                sourcePath = optarg;
                break;
            case 'k':
                kfsPath = optarg;
                break;
            case 'p':
                port = atoi(optarg);
                break;
            case 's':
                serverHost = optarg;
                break;
            case 'h':
                help = true;
                break;
            case 'v':
                verboseLogging = true;
                break;
            default:
                KFS_LOG_VA_ERROR("Unrecognized flag %c", optchar);
                help = true;
                break;
        }
    }

    if (help || (sourcePath == NULL) || (kfsPath == "") || (serverHost == "") || (port < 0)) {
        cout << "Usage: " << argv[0] << " -s <meta server name> -p <port> "
             << " -d <source path> -k <Kfs path> {-v}" << endl;
        exit(-1);
    }

    gKfsClient = getKfsClientFactory()->GetClient(serverHost, port);
    if (!gKfsClient) {
	cout << "kfs client failed to initialize...exiting" << endl;
        exit(-1);
    }

    if (verboseLogging) {
        KFS::MsgLogger::SetLevel(log4cpp::Priority::DEBUG);
    } else {
        KFS::MsgLogger::SetLevel(log4cpp::Priority::INFO);
    }

    if (stat(sourcePath, &statInfo) < 0) {
	cout << "Source path: " << sourcePath << " is non-existent!" << endl;
	exit(-1);
    }

    if (!S_ISDIR(statInfo.st_mode)) {
	BackupFile(sourcePath, kfsPath);
	exit(0);
    }

    if ((dirp = opendir(sourcePath)) == NULL) {
	perror("opendir: ");
        exit(-1);
    }

    // when doing cp -r a/b kfs://c, we need to create c/b in KFS.
    MakeKfsLeafDir(sourcePath, kfsPath);

    BackupDir(sourcePath, kfsPath);

    closedir(dirp);
}

void
MakeKfsLeafDir(const string &sourcePath, string &kfsPath)
{
    string leaf;
    string::size_type slash = sourcePath.rfind('/');

    // get everything after the last slash
    if (slash != string::npos) {
	leaf.assign(sourcePath, slash+1, string::npos);
    } else {
	leaf = sourcePath;
    }
    if (kfsPath[kfsPath.size()-1] != '/')
        kfsPath += "/";

    kfsPath += leaf;
    doMkdirs(kfsPath.c_str());
}

int
BackupFile(const string &sourcePath, const string &kfsPath)
{
    string filename;
    string::size_type slash = sourcePath.rfind('/');

    // get everything after the last slash
    if (slash != string::npos) {
	filename.assign(sourcePath, slash+1, string::npos);
    } else {
	filename = sourcePath;
    }

    // for the dest side: if kfsPath is a dir, we are copying to
    // kfsPath with srcFilename; otherwise, kfsPath is a file (that
    // potentially exists) and we are ovewriting/creating it
    if (gKfsClient->IsDirectory(kfsPath.c_str())) {
        string dst = kfsPath;

        if (dst[kfsPath.size() - 1] != '/')
            dst += "/";

        return BackupFile2(sourcePath, dst + filename);
    }

    // kfsPath is the filename that is being specified for the cp
    // target.  try to copy to there...
    return BackupFile2(sourcePath, kfsPath);
}

void
BackupDir(const string &dirname, string &kfsdirname)
{
    string subdir, kfssubdir;
    DIR *dirp;
    struct dirent *fileInfo;

    if ((dirp = opendir(dirname.c_str())) == NULL) {
        perror("opendir: ");
        exit(-1);
    }

    if (!doMkdirs(kfsdirname.c_str())) {
        cout << "Unable to make kfs dir: " << kfsdirname << endl;
        closedir(dirp);
	return;
    }

    while ((fileInfo = readdir(dirp)) != NULL) {
        if (strcmp(fileInfo->d_name, ".") == 0)
            continue;
        if (strcmp(fileInfo->d_name, "..") == 0)
            continue;

        string name = dirname + "/" + fileInfo->d_name;
        struct stat buf;
        stat(name.c_str(), &buf);

        if (S_ISDIR(buf.st_mode)) {
	    subdir = dirname + "/" + fileInfo->d_name;
            kfssubdir = kfsdirname + "/" + fileInfo->d_name;
            BackupDir(subdir, kfssubdir);
        } else if (S_ISREG(buf.st_mode)) {
	    BackupFile2(dirname + "/" + fileInfo->d_name, kfsdirname + "/" + fileInfo->d_name);
        }
    }
    closedir(dirp);
}

//
// Guts of the work to copy the file.
//
int
BackupFile2(string srcfilename, string kfsfilename)
{
    // 64MB buffers
    const int bufsize = 1 << 26;
    scoped_array<char> kfsBuf;
    int kfsfd, nRead;
    int srcFd;
    int res;

    srcFd = open(srcfilename.c_str(), O_RDONLY);
    if (srcFd  < 0) {
        cout << "Unable to open: " << srcfilename.c_str() << endl;
        exit(-1);
    }

    kfsfd = gKfsClient->Create((char *) kfsfilename.c_str());
    if (kfsfd < 0) {
        cout << "Create " << kfsfilename << " failed: " << kfsfd << endl;
        exit(-1);
    }
    kfsBuf.reset(new char[bufsize]);

    while (1) {
        nRead = read(srcFd, kfsBuf.get(), bufsize);

        if (nRead <= 0)
            break;

        res = gKfsClient->Write(kfsfd, kfsBuf.get(), nRead);
        if (res < 0) {
            cout << "Write failed with error code: " << res << endl;
            exit(-1);
        }
    }
    close(srcFd);
    gKfsClient->Close(kfsfd);

    return 0;
}

int
BackupFile3(string srcfilename, string kfsfilename)
{
    // 64MB buffers
    const int bufsize = 1 << 26;
    scoped_array<char> kfsBuf;
    int kfsfd, nRead;
    long long n = 0;
    ifstream ifs;
    int res;

    ifs.open(srcfilename.c_str(), ios_base::in);
    if (!ifs) {
        cout << "Unable to open: " << srcfilename.c_str() << endl;
        exit(-1);
    }

    kfsfd = gKfsClient->Create((char *) kfsfilename.c_str());
    if (kfsfd < 0) {
        cout << "Create " << kfsfilename << " failed: " << kfsfd << endl;
        exit(-1);
    }
    kfsBuf.reset(new char[bufsize]);

    while (!ifs.eof()) {
        ifs.read(kfsBuf.get(), bufsize);

        nRead = ifs.gcount();

        if (nRead <= 0)
            break;

        res = gKfsClient->Write(kfsfd, kfsBuf.get(), nRead);
        if (res < 0) {
            cout << "Write failed with error code: " << res << endl;
            exit(-1);
        }
        n += nRead;
        // cout << "Wrote: " << n << endl;
    }
    ifs.close();
    gKfsClient->Close(kfsfd);

    return 0;
}

bool
doMkdirs(const char *path)
{
    int res;

    res = gKfsClient->Mkdirs((char *) path);
    if ((res < 0) && (res != -EEXIST)) {
        cout << "Mkdir failed: " << ErrorCodeToStr(res) << endl;
        return false;
    }
    return true;
}

ext: cc


text:
#ifndef HERMIT_SPIRIT_QI_JSON_HPP
#define HERMIT_SPIRIT_QI_JSON_HPP

#include <utility>
#include <string>
#include <vector>
#include <map>
#include <boost/variant.hpp>
#include <boost/variant/recursive_variant.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/fusion/adapted/std_pair.hpp>
#include <boost/fusion/include/std_pair.hpp>

#include <hermit/json.hpp>
#include <hermit/format/read/utf8.hpp>
#include <hermit/format/read/utf16.hpp>
#include <hermit/format/write/utf8.hpp>

namespace hermit {
  namespace spirit {
    namespace qi {
      boost::spirit::qi::uint_parser<uint8_t, 16, 2, 2> hex2_p;

      template< typename Iterator >
      class json_encoding :
        public boost::spirit::qi::grammar<
          Iterator,
          hermit::json_encoding_type()
        > {
      public:
        json_encoding() : json_encoding::base_type( root ) {
          namespace qi = boost::spirit::qi;
          namespace phx = boost::phoenix;
          root = ( qi::byte_ >> qi::byte_ >> qi::byte_ >> qi::byte_ )[
            phx::if_( qi::_1 == 0 )[
              phx::if_( qi::_2 == 0 )[
                phx::if_( qi::_3 == 0 )[
                  qi::_val = hermit::json_utf32be
                ].else_[
                  qi::_val = hermit::json_utf32
                ]
              ].else_[
                qi::_val = hermit::json_utf16be
              ]
            ].else_[
              phx::if_( qi::_2 == 0 )[
                phx::if_( qi::_3 == 0 )[
                  qi::_val = hermit::json_utf32le
                ].else_[
                  qi::_val = hermit::json_utf16le
                ]
              ].else_[
                phx::if_( qi::_3 == 0 )[
                  phx::if_( qi::_4 == 0 )[
                    qi::_val = hermit::json_utf32
                  ].else_[
                    qi::_val = hermit::json_utf16
                  ]
                ].else_[
                  phx::if_( qi::_4 == 0 )[
                    qi::_val = hermit::json_utf16
                  ].else_[
                    qi::_val = hermit::json_utf8
                  ]
                ]
              ]
            ]
          ];
        }
      private:
        boost::spirit::qi::rule< Iterator, hermit::json_encoding_type() > root;
      };

      template< typename Iterator >
      class json :
        public boost::spirit::qi::grammar<
          Iterator,
          hermit::json()
        > {
      public:
        json() : json::base_type( root ) {
          namespace qi = boost::spirit::qi;
          namespace phx = boost::phoenix;
          null_ = qi::lit( "null" )[ qi::_val = none_type() ];
          root = qi::omit[ *qi::standard::space ] >> (object|array) >> qi::omit[ *qi::standard::space ];
          value_ = string_|qi::double_|object|array|qi::bool_|null_;
          escape_sequence = '\\' >> (
            qi::standard::char_( '"' )[ qi::_val = '"' ] |
            qi::standard::char_( '\\' )[ qi::_val = '\\' ] |
            qi::standard::char_( '/' )[ qi::_val = '/' ] |
            qi::standard::char_( 'b' )[ qi::_val = '\b' ] |
            qi::standard::char_( 'n' )[ qi::_val = '\n' ] |
            qi::standard::char_( 'f' )[ qi::_val = '\f' ] |
            qi::standard::char_( 'r' )[ qi::_val = '\r' ] |
            qi::standard::char_( 't' )[ qi::_val = '\t' ]
          );
          unicode_string = +(
            "\\u" >> hex2_p >> hex2_p
          )[
            phx::push_back( qi::_a, qi::_1 ),
            phx::push_back( qi::_a, qi::_2 )
          ] >> qi::eps[
            qi::_val = phx::bind( &json::utf16_to_utf8, qi::_a, qi::_pass )
          ];
          string_ = string_utf8[
            qi::_val = phx::bind( &json::utf8_to_ucs4, qi::_1, qi::_pass )
          ];
          string_utf8 = '"' >> *( (
            qi::repeat(1)[
              ( qi::standard::char_ - qi::standard::cntrl - '\\' - '"' )|
              escape_sequence
            ] )[
              phx::insert(
                qi::_val,
                phx::end( qi::_val ),
                phx::begin( qi::_1 ),
                phx::end( qi::_1 )
              )
            ]|unicode_string[
              phx::insert(
                qi::_val,
                phx::end( qi::_val ),
                phx::begin( qi::_1 ),
                phx::end( qi::_1 )
              )
            ] ) >> '"';
          array = qi::skip( qi::standard::space )[ '[' >> -( value_ % ',' ) >>  ']' ];
          named_value = qi::skip( qi::standard::space )[ ( string_ >> ':' >> value_ ) ];
          object = qi::skip( qi::standard::space )[ '{' >> -( named_value % ',' ) >> '}' ];
        }
      private:
        static std::string utf16_to_utf8( const std::vector< uint8_t > &src, bool &_pass ) {
          const auto ucs4_string = hermit::format::read_utf16be( src );
          if( !ucs4_string ) {
            _pass = false;
            return "";
          }
          const auto utf8_string = hermit::format::write_utf8( *ucs4_string );
          if( !utf8_string ) {
            _pass = false;
            return "";
          }
          _pass = true;
          return *utf8_string;
        }
        static std::u32string utf8_to_ucs4( const std::string &src, bool &_pass ) {
          const auto ucs4_string = hermit::format::read_utf8( src );
          if( !ucs4_string ) {
            _pass = false;
            return std::u32string();
          }
          _pass = true;
          return *ucs4_string;
        }
        boost::spirit::qi::rule< Iterator, hermit::json() > root;
        boost::spirit::qi::rule< Iterator, char() > escape_sequence;
        boost::spirit::qi::rule< Iterator, std::string() > string_utf8;
        boost::spirit::qi::rule< Iterator, std::u32string() > string_;
        boost::spirit::qi::rule< Iterator, std::string(), boost::spirit::qi::locals< std::vector< uint8_t > > > unicode_string;
        boost::spirit::qi::rule< Iterator, none_type() > null_;
        boost::spirit::qi::rule< Iterator, hermit::json() > value_;
        boost::spirit::qi::rule< Iterator, std::vector< hermit::json >() > array;
        boost::spirit::qi::rule< Iterator, std::pair< std::u32string, hermit::json >() > named_value;
        boost::spirit::qi::rule< Iterator, std::map< std::u32string, hermit::json >() > object;
      };
    }
  }
}

#endif

ext: hpp

max_stars_repo_name: Fadis/hermit

max_issues_repo_name: Fadis/hermit

max_forks_repo_name: Fadis/hermit


text:
//==============================================================================
//         Copyright 2003 - 2012 LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2012 LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#define NT2_UNIT_MODULE "nt2 operator toolbox - unaligned_load/scalar Mode"

#include <boost/simd/include/functions/unaligned_load.hpp>
#include <boost/simd/sdk/simd/native.hpp>
#include <boost/fusion/include/at_c.hpp>
#include <boost/fusion/include/make_vector.hpp>
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/simd/sdk/config/types.hpp>
#include <boost/simd/sdk/config/type_lists.hpp>
#include <nt2/sdk/unit/module.hpp>
#include <nt2/sdk/unit/tests/relation.hpp>

NT2_TEST_CASE_TPL(unaligned_load, BOOST_SIMD_TYPES)
{
  using boost::simd::unaligned_load;

  T data[5] = {0,1,2,3,4};

  NT2_TEST_EQUAL( (unaligned_load<T,-4>(&data[0],4)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,-4>(&data[0],5)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,-4>(&data[0],6)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,-4>(&data[0],7)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,-4>(&data[0],8)), T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,-3>(&data[0],3)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,-3>(&data[0],4)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,-3>(&data[0],5)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,-3>(&data[0],6)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,-3>(&data[0],7)), T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,-2>(&data[0],2)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,-2>(&data[0],3)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,-2>(&data[0],4)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,-2>(&data[0],5)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,-2>(&data[0],6)), T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,-1>(&data[0],1)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,-1>(&data[0],2)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,-1>(&data[0],3)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,-1>(&data[0],4)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,-1>(&data[0],5)), T(4) );

  NT2_TEST_EQUAL( unaligned_load<T>(&data[0],0), T(0) );
  NT2_TEST_EQUAL( unaligned_load<T>(&data[0],1), T(1) );
  NT2_TEST_EQUAL( unaligned_load<T>(&data[0],2), T(2) );
  NT2_TEST_EQUAL( unaligned_load<T>(&data[0],3), T(3) );
  NT2_TEST_EQUAL( unaligned_load<T>(&data[0],4), T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,0>(&data[0],0)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,0>(&data[0],1)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,0>(&data[0],2)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,0>(&data[0],3)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,0>(&data[0],4)), T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,1>(&data[0],-1)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,1>(&data[0],0)) , T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,1>(&data[0],1)) , T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,1>(&data[0],2)) , T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,1>(&data[0],3)) , T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,2>(&data[0],-2)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,2>(&data[0],-1)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,2>(&data[0],0)) , T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,2>(&data[0],1)) , T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,2>(&data[0],2)) , T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,3>(&data[0],-3)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,3>(&data[0],-2)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,3>(&data[0],-1)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,3>(&data[0],0)) , T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,3>(&data[0],1)) , T(4) );

  NT2_TEST_EQUAL( (unaligned_load<T,4>(&data[0],-4)), T(0) );
  NT2_TEST_EQUAL( (unaligned_load<T,4>(&data[0],-3)), T(1) );
  NT2_TEST_EQUAL( (unaligned_load<T,4>(&data[0],-2)), T(2) );
  NT2_TEST_EQUAL( (unaligned_load<T,4>(&data[0],-1)), T(3) );
  NT2_TEST_EQUAL( (unaligned_load<T,4>(&data[0],0)) , T(4) );
}

NT2_TEST_CASE( unaligned_load_sequence )
{
  using boost::simd::unaligned_load;
  using boost::simd::tag::unaligned_load_;
  using boost::simd::native;
  using boost::simd::meta::cardinal_of;
  typedef BOOST_SIMD_DEFAULT_EXTENSION  ext_t;

  double d = 3.4;
  float  f = 1.8f;
  char   c = 'a';

  boost::fusion::vector<double,float,char> v;

  v = unaligned_load< boost::fusion::vector<double,float,char> >(boost::fusion::make_vector(&d, &f, &c), 0);

  NT2_TEST_EQUAL(boost::fusion::at_c<0>(v) , d);
  NT2_TEST_EQUAL(boost::fusion::at_c<1>(v) , f);
  NT2_TEST_EQUAL(boost::fusion::at_c<2>(v) , c);
}

NT2_TEST_CASE( unaligned_load_pointer_of_sequence )
{
  using boost::simd::unaligned_load;
  using boost::simd::tag::unaligned_load_;
  typedef BOOST_SIMD_DEFAULT_EXTENSION  ext_t;

  double d = 3.4;
  float  f = 1.8f;
  char   c = 'a';

  boost::fusion::vector<double,float,char> s(d,f,c);
  boost::fusion::vector<double,float,char> v;

  v = unaligned_load< boost::fusion::vector<double,float,char> >(&s, 0);

  NT2_TEST_EQUAL(boost::fusion::at_c<0>(v) , d);
  NT2_TEST_EQUAL(boost::fusion::at_c<1>(v) , f);
  NT2_TEST_EQUAL(boost::fusion::at_c<2>(v) , c);
}