tokenizer.h

Go to the documentation of this file.

/*******************************************************************************
** Copyright (c) 2017 MAK Technologies, Inc.
** All rights reserved.
*******************************************************************************/



#pragma once
#include <vrfutil/vrfutilDefines.h>

#include <fstream>
#include <string>
#include <vector>



using namespace std;

class DT_DLL_vrfutil DtTokenizer
{
public:
   DtTokenizer();
   ~DtTokenizer();

   void setFilename( const string& filename );
   void setStream( istream& stream );

   void setRecordSeparator( const string& separator );
   void setFieldSeparators( const string& separators );
   void setQuoteChars( const string& quoteChars );
   void setEscapeChars( const string& escapeChars );
   void setSeparatorData( const string& recordSeparator, const string& fieldSeparators = ",",
                          const string& quoteChars = "\"", const string& escapeChars = "\\" );

   void setTrimWhitespace( bool bTrim );
   void setAddEmptyTokens( bool bAdd );

   void setSkipNextRecord( bool bSkip );

   bool nextRecord( vector<string>& record );

protected:
   virtual bool isEndOfRecord( const string& str );

private:
   void readRecord( string& record );

   string myRecordSeparator;
   string myFieldSeparators;
   string myQuoteChars;
   string myEscapeChars;

   bool myTrimWhitespace;
   bool myAddEmptyTokens;
   bool mySkipNextRecord;

   ifstream myFileStream;
   istream* pMyStream;
};