reader.h

Go to the documentation of this file.

#ifndef RAPIDJSON_READER_H_
#define RAPIDJSON_READER_H_

// Copyright (c) 2011 Milo Yip (miloyip@gmail.com)
// Version 0.1

#include "rapidjson.h"
#include "internal/pow10.h"
#include "internal/stack.h"
#include <csetjmp>

#ifdef RAPIDJSON_SSE42
#include <nmmintrin.h>
#elif defined(RAPIDJSON_SSE2)
#include <emmintrin.h>
#endif

#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4127) // conditional expression is constant
#endif

#ifndef RAPIDJSON_PARSE_ERROR
#define RAPIDJSON_PARSE_ERROR(msg, offset) \
   RAPIDJSON_MULTILINEMACRO_BEGIN \
   parseError_ = msg; \
   errorOffset_ = offset; \
   longjmp(jmpbuf_, 1); \
   RAPIDJSON_MULTILINEMACRO_END
#endif

namespace rapidjson {

// ParseFlag

enum ParseFlag {
   kParseDefaultFlags = 0,       
   kParseInsituFlag = 1       
};

// Handler


// BaseReaderHandler


template<typename Encoding = UTF8<> >
struct BaseReaderHandler {
   typedef typename Encoding::Ch Ch;

   void Default() {}
   void Null() { Default(); }
   void Bool(bool) { Default(); }
   void Int(int) { Default(); }
   void Uint(unsigned) { Default(); }
   void Int64(int64_t) { Default(); }
   void Uint64(uint64_t) { Default(); }
   void Double(double) { Default(); }
   void String(const Ch*, SizeType, bool) { Default(); }
   void StartObject() { Default(); }
   void EndObject(SizeType) { Default(); }
   void StartArray() { Default(); }
   void EndArray(SizeType) { Default(); }
};

// SkipWhitespace


template<typename Stream>
void SkipWhitespace(Stream& stream) {
   Stream s = stream;   // Use a local copy for optimization
   while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t')
      s.Take();
   stream = s;
}

#ifdef RAPIDJSON_SSE42

inline const char *SkipWhitespace_SIMD(const char* p) {
   static const char whitespace[16] = " \n\r\t";
   __m128i w = _mm_loadu_si128((const __m128i *)&whitespace[0]);

   for (;;) {
      __m128i s = _mm_loadu_si128((const __m128i *)p);
      unsigned r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY));
      if (r == 0) // all 16 characters are whitespace
         p += 16;
      else {      // some of characters may be non-whitespace
#ifdef _MSC_VER      // Find the index of first non-whitespace
         unsigned long offset;
         if (_BitScanForward(&offset, r))
            return p + offset;
#else
         if (r != 0)
            return p + __builtin_ffs(r) - 1;
#endif
      }
   }
}

#elif defined(RAPIDJSON_SSE2)

inline const char *SkipWhitespace_SIMD(const char* p) {
   static const char whitespaces[4][17] = {
      "                ",
      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
      "\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r\r",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"};

   __m128i w0 = _mm_loadu_si128((const __m128i *)&whitespaces[0][0]);
   __m128i w1 = _mm_loadu_si128((const __m128i *)&whitespaces[1][0]);
   __m128i w2 = _mm_loadu_si128((const __m128i *)&whitespaces[2][0]);
   __m128i w3 = _mm_loadu_si128((const __m128i *)&whitespaces[3][0]);

   for (;;) {
      __m128i s = _mm_loadu_si128((const __m128i *)p);
      __m128i x = _mm_cmpeq_epi8(s, w0);
      x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1));
      x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2));
      x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3));
      unsigned short r = ~_mm_movemask_epi8(x);
      if (r == 0) // all 16 characters are whitespace
         p += 16;
      else {      // some of characters may be non-whitespace
#ifdef _MSC_VER      // Find the index of first non-whitespace
         unsigned long offset;
         if (_BitScanForward(&offset, r))
            return p + offset;
#else
         if (r != 0)
            return p + __builtin_ffs(r) - 1;
#endif
      }
   }
}

#endif // RAPIDJSON_SSE2

#ifdef RAPIDJSON_SIMD

template<> inline void SkipWhitespace(InsituStringStream& stream) { 
   stream.src_ = const_cast<char*>(SkipWhitespace_SIMD(stream.src_));
}

template<> inline void SkipWhitespace(StringStream& stream) {
   stream.src_ = SkipWhitespace_SIMD(stream.src_);
}
#endif // RAPIDJSON_SIMD

// GenericReader


template <typename Encoding, typename Allocator = MemoryPoolAllocator<> >
class GenericReader {
public:
   typedef typename Encoding::Ch Ch;


   GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseError_(0), errorOffset_(0) {}


   template <unsigned parseFlags, typename Stream, typename Handler>
   bool Parse(Stream& stream, Handler& handler) {
      parseError_ = 0;
      errorOffset_ = 0;

#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4611) // interaction between '_setjmp' and C++ object destruction is non-portable
#endif
      if (setjmp(jmpbuf_)) {
#ifdef _MSC_VER
#pragma warning(pop)
#endif
         stack_.Clear();
         return false;
      }

      SkipWhitespace(stream);

      if (stream.Peek() == '\0')
         RAPIDJSON_PARSE_ERROR("Text only contains white space(s)", stream.Tell());
      else {
         switch (stream.Peek()) {
            case '{': ParseObject<parseFlags>(stream, handler); break;
            case '[': ParseArray<parseFlags>(stream, handler); break;
            default: RAPIDJSON_PARSE_ERROR("Expect either an object or array at root", stream.Tell());
         }
         SkipWhitespace(stream);

         if (stream.Peek() != '\0')
            RAPIDJSON_PARSE_ERROR("Nothing should follow the root object or array.", stream.Tell());
      }

      return true;
   }

   bool HasParseError() const { return parseError_ != 0; }
   const char* GetParseError() const { return parseError_; }
   size_t GetErrorOffset() const { return errorOffset_; }

private:
   // Parse object: { string : value, ... }
   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseObject(Stream& stream, Handler& handler) {
      RAPIDJSON_ASSERT(stream.Peek() == '{');
      stream.Take(); // Skip '{'
      handler.StartObject();
      SkipWhitespace(stream);

      if (stream.Peek() == '}') {
         stream.Take();
         handler.EndObject(0);   // empty object
         return;
      }

      for (SizeType memberCount = 0;;) {
         if (stream.Peek() != '"') {
            RAPIDJSON_PARSE_ERROR("Name of an object member must be a string", stream.Tell());
            break;
         }

         ParseString<parseFlags>(stream, handler);
         SkipWhitespace(stream);

         if (stream.Take() != ':') {
            RAPIDJSON_PARSE_ERROR("There must be a colon after the name of object member", stream.Tell());
            break;
         }
         SkipWhitespace(stream);

         ParseValue<parseFlags>(stream, handler);
         SkipWhitespace(stream);

         ++memberCount;

         switch(stream.Take()) {
            case ',': SkipWhitespace(stream); break;
            case '}': handler.EndObject(memberCount); return;
            default:  RAPIDJSON_PARSE_ERROR("Must be a comma or '}' after an object member", stream.Tell());
         }
      }
   }

   // Parse array: [ value, ... ]
   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseArray(Stream& stream, Handler& handler) {
      RAPIDJSON_ASSERT(stream.Peek() == '[');
      stream.Take(); // Skip '['
      handler.StartArray();
      SkipWhitespace(stream);

      if (stream.Peek() == ']') {
         stream.Take();
         handler.EndArray(0); // empty array
         return;
      }

      for (SizeType elementCount = 0;;) {
         ParseValue<parseFlags>(stream, handler);
         ++elementCount;
         SkipWhitespace(stream);

         switch (stream.Take()) {
            case ',': SkipWhitespace(stream); break;
            case ']': handler.EndArray(elementCount); return;
            default:  RAPIDJSON_PARSE_ERROR("Must be a comma or ']' after an array element.", stream.Tell());
         }
      }
   }

   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseNull(Stream& stream, Handler& handler) {
      RAPIDJSON_ASSERT(stream.Peek() == 'n');
      stream.Take();

      if (stream.Take() == 'u' && stream.Take() == 'l' && stream.Take() == 'l')
         handler.Null();
      else
         RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1);
   }

   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseTrue(Stream& stream, Handler& handler) {
      RAPIDJSON_ASSERT(stream.Peek() == 't');
      stream.Take();

      if (stream.Take() == 'r' && stream.Take() == 'u' && stream.Take() == 'e')
         handler.Bool(true);
      else
         RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell());
   }

   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseFalse(Stream& stream, Handler& handler) {
      RAPIDJSON_ASSERT(stream.Peek() == 'f');
      stream.Take();

      if (stream.Take() == 'a' && stream.Take() == 'l' && stream.Take() == 's' && stream.Take() == 'e')
         handler.Bool(false);
      else
         RAPIDJSON_PARSE_ERROR("Invalid value", stream.Tell() - 1);
   }

   // Helper function to parse four hexidecimal digits in \uXXXX in ParseString().
   template<typename Stream>
   unsigned ParseHex4(Stream& stream) {
      Stream s = stream;   // Use a local copy for optimization
      unsigned codepoint = 0;
      for (int i = 0; i < 4; i++) {
         Ch c = s.Take();
         codepoint <<= 4;
         codepoint += c;
         if (c >= '0' && c <= '9')
            codepoint -= '0';
         else if (c >= 'A' && c <= 'F')
            codepoint -= 'A' - 10;
         else if (c >= 'a' && c <= 'f')
            codepoint -= 'a' - 10;
         else 
            RAPIDJSON_PARSE_ERROR("Incorrect hex digit after \\u escape", s.Tell() - 1);
      }
      stream = s; // Restore stream
      return codepoint;
   }

   // Parse string, handling the prefix and suffix double quotes and escaping.
   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseString(Stream& stream, Handler& handler) {
#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
      static const Ch escape[256] = {
         Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', 
         Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, 
         0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, 
         0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
         Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16
      };
#undef Z16

      Stream s = stream;   // Use a local copy for optimization
      RAPIDJSON_ASSERT(s.Peek() == '\"');
      s.Take();   // Skip '\"'
      Ch *head;
      SizeType len;
      if (parseFlags & kParseInsituFlag)
         head = s.PutBegin();
      else
         len = 0;

#define RAPIDJSON_PUT(x) \
   do { \
      if (parseFlags & kParseInsituFlag) \
         s.Put(x); \
      else { \
         *stack_.template Push<Ch>() = x; \
         ++len; \
      } \
   } while(false)

      for (;;) {
         Ch c = s.Take();
         if (c == '\\') {  // Escape
            Ch e = s.Take();
            if ((sizeof(Ch) == 1 || e < 256) && escape[(unsigned char)e])
               RAPIDJSON_PUT(escape[(unsigned char)e]);
            else if (e == 'u') { // Unicode
               unsigned codepoint = ParseHex4(s);
               if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { // Handle UTF-16 surrogate pair
                  if (s.Take() != '\\' || s.Take() != 'u') {
                     RAPIDJSON_PARSE_ERROR("Missing the second \\u in surrogate pair", s.Tell() - 2);
                     return;
                  }
                  unsigned codepoint2 = ParseHex4(s);
                  if (codepoint2 < 0xDC00 || codepoint2 > 0xDFFF) {
                     RAPIDJSON_PARSE_ERROR("The second \\u in surrogate pair is invalid", s.Tell() - 2);
                     return;
                  }
                  codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000;
               }

               Ch buffer[4];
               SizeType count = SizeType(Encoding::Encode(buffer, codepoint) - &buffer[0]);

               if (parseFlags & kParseInsituFlag) 
                  for (SizeType i = 0; i < count; i++)
                     s.Put(buffer[i]);
               else {
                  memcpy(stack_.template Push<Ch>(count), buffer, count * sizeof(Ch));
                  len += count;
               }
            }
            else {
               RAPIDJSON_PARSE_ERROR("Unknown escape character", stream.Tell() - 1);
               return;
            }
         }
         else if (c == '"') { // Closing double quote
            if (parseFlags & kParseInsituFlag) {
               size_t length = s.PutEnd(head);
               RAPIDJSON_ASSERT(length <= 0xFFFFFFFF);
               RAPIDJSON_PUT('\0'); // null-terminate the string
               handler.String(head, SizeType(length), false);
            }
            else {
               RAPIDJSON_PUT('\0');
               handler.String(stack_.template Pop<Ch>(len), len - 1, true);
            }
            stream = s; // restore stream
            return;
         }
         else if (c == '\0') {
            RAPIDJSON_PARSE_ERROR("lacks ending quotation before the end of string", stream.Tell() - 1);
            return;
         }
         else if ((unsigned)c < 0x20) {   // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
            RAPIDJSON_PARSE_ERROR("Incorrect unescaped character in string", stream.Tell() - 1);
            return;
         }
         else
            RAPIDJSON_PUT(c); // Normal character, just copy
      }
#undef RAPIDJSON_PUT
   }

   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseNumber(Stream& stream, Handler& handler) {
      Stream s = stream; // Local copy for optimization
      // Parse minus
      bool minus = false;
      if (s.Peek() == '-') {
         minus = true;
         s.Take();
      }

      // Parse int: zero / ( digit1-9 *DIGIT )
      unsigned i;
      bool try64bit = false;
      if (s.Peek() == '0') {
         i = 0;
         s.Take();
      }
      else if (s.Peek() >= '1' && s.Peek() <= '9') {
         i = s.Take() - '0';

         if (minus)
            while (s.Peek() >= '0' && s.Peek() <= '9') {
               if (i >= 214748364) { // 2^31 = 2147483648
                  if (i != 214748364 || s.Peek() > '8') {
                     try64bit = true;
                     break;
                  }
               }
               i = i * 10 + (s.Take() - '0');
            }
         else
            while (s.Peek() >= '0' && s.Peek() <= '9') {
               if (i >= 429496729) { // 2^32 - 1 = 4294967295
                  if (i != 429496729 || s.Peek() > '5') {
                     try64bit = true;
                     break;
                  }
               }
               i = i * 10 + (s.Take() - '0');
            }
      }
      else {
         RAPIDJSON_PARSE_ERROR("Expect a value here.", stream.Tell());
         return;
      }

      // Parse 64bit int
      uint64_t i64 = 0;
      bool useDouble = false;
      if (try64bit) {
         i64 = i;
         if (minus) 
            while (s.Peek() >= '0' && s.Peek() <= '9') {             
               if (i64 >= 922337203685477580uLL) // 2^63 = 9223372036854775808
                  if (i64 != 922337203685477580uLL || s.Peek() > '8') {
                     useDouble = true;
                     break;
                  }
               i64 = i64 * 10 + (s.Take() - '0');
            }
         else
            while (s.Peek() >= '0' && s.Peek() <= '9') {             
               if (i64 >= 1844674407370955161uLL) // 2^64 - 1 = 18446744073709551615
                  if (i64 != 1844674407370955161uLL || s.Peek() > '5') {
                     useDouble = true;
                     break;
                  }
               i64 = i64 * 10 + (s.Take() - '0');
            }
      }

      // Force double for big integer
      double d = 0.0;
      if (useDouble) {
         d = (double)i64;
         while (s.Peek() >= '0' && s.Peek() <= '9') {
            if (d >= 1E307) {
               RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell());
               return;
            }
            d = d * 10 + (s.Take() - '0');
         }
      }

      // Parse frac = decimal-point 1*DIGIT
      int expFrac = 0;
      if (s.Peek() == '.') {
         if (!useDouble) {
            d = try64bit ? (double)i64 : (double)i;
            useDouble = true;
         }
         s.Take();

         if (s.Peek() >= '0' && s.Peek() <= '9') {
            d = d * 10 + (s.Take() - '0');
            --expFrac;
         }
         else {
            RAPIDJSON_PARSE_ERROR("At least one digit in fraction part", stream.Tell());
            return;
         }

         while (s.Peek() >= '0' && s.Peek() <= '9') {
            if (expFrac > -16) {
               d = d * 10 + (s.Peek() - '0');
               --expFrac;
            }
            s.Take();
         }
      }

      // Parse exp = e [ minus / plus ] 1*DIGIT
      int exp = 0;
      if (s.Peek() == 'e' || s.Peek() == 'E') {
         if (!useDouble) {
            d = try64bit ? (double)i64 : (double)i;
            useDouble = true;
         }
         s.Take();

         bool expMinus = false;
         if (s.Peek() == '+')
            s.Take();
         else if (s.Peek() == '-') {
            s.Take();
            expMinus = true;
         }

         if (s.Peek() >= '0' && s.Peek() <= '9') {
            exp = s.Take() - '0';
            while (s.Peek() >= '0' && s.Peek() <= '9') {
               exp = exp * 10 + (s.Take() - '0');
               if (exp > 308) {
                  RAPIDJSON_PARSE_ERROR("Number too big to store in double", stream.Tell());
                  return;
               }
            }
         }
         else {
            RAPIDJSON_PARSE_ERROR("At least one digit in exponent", s.Tell());
            return;
         }

         if (expMinus)
            exp = -exp;
      }

      // Finish parsing, call event according to the type of number.
      if (useDouble) {
         d *= internal::Pow10(exp + expFrac);
         handler.Double(minus ? -d : d);
      }
      else {
         if (try64bit) {
            if (minus)
               handler.Int64(-(int64_t)i64);
            else
               handler.Uint64(i64);
         }
         else {
            if (minus)
               handler.Int(-(int)i);
            else
               handler.Uint(i);
         }
      }

      stream = s; // restore stream
   }

   // Parse any JSON value
   template<unsigned parseFlags, typename Stream, typename Handler>
   void ParseValue(Stream& stream, Handler& handler) {
      switch (stream.Peek()) {
         case 'n': ParseNull  <parseFlags>(stream, handler); break;
         case 't': ParseTrue  <parseFlags>(stream, handler); break;
         case 'f': ParseFalse <parseFlags>(stream, handler); break;
         case '"': ParseString<parseFlags>(stream, handler); break;
         case '{': ParseObject<parseFlags>(stream, handler); break;
         case '[': ParseArray <parseFlags>(stream, handler); break;
         default : ParseNumber<parseFlags>(stream, handler);
      }
   }

   static const size_t kDefaultStackCapacity = 256;   
   internal::Stack<Allocator> stack_;  
   jmp_buf jmpbuf_;              
   const char* parseError_;
   size_t errorOffset_;
}; // class GenericReader

typedef GenericReader<UTF8<> > Reader;

} // namespace rapidjson

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#endif // RAPIDJSON_READER_H_