[jackson-dataformat-smile] 01/02: Imported Upstream version 2.2.2
Wolodja Wentland
babilen-guest at alioth.debian.org
Fri Aug 23 16:15:01 UTC 2013
This is an automated email from the git hooks/post-receive script.
babilen-guest pushed a commit to branch master
in repository jackson-dataformat-smile.
commit 519b45a0637a0dc739363776b9e9ef4b1bc665da
Author: Wolodja Wentland <babilen at gmail.com>
Date: Sun Aug 11 13:21:48 2013 +0200
Imported Upstream version 2.2.2
---
.gitignore | 21 +
README.md | 38 +
pom.xml | 88 +
release-notes/CREDITS | 11 +
release-notes/VERSION | 96 +
.../dataformat/smile/PackageVersion.java.in | 20 +
.../dataformat/smile/SmileBufferRecycler.java | 51 +
.../jackson/dataformat/smile/SmileConstants.java | 367 +++
.../jackson/dataformat/smile/SmileFactory.java | 564 +++++
.../jackson/dataformat/smile/SmileGenerator.java | 2298 +++++++++++++++++
.../jackson/dataformat/smile/SmileParser.java | 2673 ++++++++++++++++++++
.../dataformat/smile/SmileParserBootstrapper.java | 262 ++
.../jackson/dataformat/smile/SmileUtil.java | 46 +
.../fasterxml/jackson/dataformat/smile/Tool.java | 160 ++
.../smile/async/NonBlockingInputFeeder.java | 48 +
.../dataformat/smile/async/NonBlockingParser.java | 31 +
.../smile/async/NonBlockingParserImpl.java | 2227 ++++++++++++++++
.../dataformat/smile/async/package-info.java | 6 +
.../jackson/dataformat/smile/package-info.java | 7 +
.../com.fasterxml.jackson.core.JsonFactory | 15 +
.../jackson/dataformat/smile/SmileTestBase.java | 223 ++
.../jackson/dataformat/smile/TestDocBoundary.java | 109 +
.../jackson/dataformat/smile/TestFeatures.java | 37 +
.../dataformat/smile/TestFormatDetection.java | 44 +
.../jackson/dataformat/smile/TestGenerator.java | 246 ++
.../dataformat/smile/TestGeneratorBinary.java | 97 +
.../smile/TestGeneratorBufferRecycle.java | 76 +
.../dataformat/smile/TestGeneratorLongStrings.java | 94 +
.../dataformat/smile/TestGeneratorNumbers.java | 131 +
.../dataformat/smile/TestGeneratorSymbols.java | 241 ++
.../dataformat/smile/TestGeneratorWithRawUtf8.java | 225 ++
.../smile/TestGeneratorWithSerializedString.java | 86 +
.../jackson/dataformat/smile/TestMapper.java | 42 +
.../jackson/dataformat/smile/TestParser.java | 558 ++++
.../jackson/dataformat/smile/TestParserBinary.java | 214 ++
.../dataformat/smile/TestParserLocation.java | 60 +
.../jackson/dataformat/smile/TestParserNames.java | 66 +
.../dataformat/smile/TestParserNumbers.java | 303 +++
.../dataformat/smile/TestParserSymbolHandling.java | 564 +++++
.../dataformat/smile/TestParserUnicode.java | 27 +
.../dataformat/smile/TestSmileDetection.java | 151 ++
.../dataformat/smile/TestSmileGeneratorBinary.java | 97 +
.../jackson/dataformat/smile/TestSmileUtil.java | 44 +
.../jackson/dataformat/smile/TestTreeHandling.java | 37 +
.../jackson/dataformat/smile/TestVersions.java | 32 +
45 files changed, 12833 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..84914ec
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,21 @@
+# use glob syntax.
+syntax: glob
+*.class
+*~
+*.bak
+*.off
+*.old
+.DS_Store
+
+# building
+target
+
+# Eclipse
+.classpath
+.project
+.settings
+
+# IDEA
+*.iml
+*.ipr
+*.iws
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..b00378a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,38 @@
+## Overview
+
+This Jackson extension handles reading and writing of data using [Smile](http://wiki.fasterxml.com/SmileFormatSpec) data format ("binary JSON").
+It extends standard Jackson streaming API (`JsonFactory`, `JsonParser`, `JsonGenerator`), and as such works seamlessly with all the higher level data abstractions (data binding, tree model, and pluggable extensions).
+
+[](https://fasterxml.ci.cloudbees.com/job/jackson-dataformat-smile-master/)
+
+## Status
+
+Module is fully usable, but depends on Jackson 2.0; older 1.x version is still available via Codehaus, and Jackson 1.x Download page.
+
+## Maven dependency
+
+To use this extension on Maven-based projects, use following dependency:
+
+```xml
+<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-smile</artifactId>
+ <version>2.1.1</version>
+</dependency>
+```
+
+(or whatever version is most up-to-date at the moment)
+
+## Usage
+
+Basic usage is by using `SmileFactory` in places where you would usually use `JsonFactory`:
+
+```java
+SmileFactory f = new SmileFactory();
+// can configure instance with 'SmileParser.Feature' and 'SmileGenerator.Feature'
+ObjectMapper mapper = new ObjectMapper(f);
+// and then read/write data as usual
+SomeType value = ...;
+byte[] smileData = mapper.writeValueAsBytes(value);
+SomeType otherValue = mapper.readValue(smileData, SomeType.class);
+```
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..69a8914
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>com.fasterxml</groupId>
+ <artifactId>oss-parent</artifactId>
+ <version>10</version>
+ </parent>
+
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-smile</artifactId>
+ <version>2.2.2</version>
+
+ <name>Jackson-dataformat-Smile</name>
+ <description>Support for reading and writing Smile ("binary JSON")
+encoded data using Jackson abstractions (streaming API, data binding,
+tree model)
+ </description>
+ <url>http://wiki.fasterxml.com/JacksonForSmile</url>
+
+ <scm>
+ <connection>scm:git:git at github.com:FasterXML/jackson-dataformat-smile.git</connection>
+ <developerConnection>scm:git:git at github.com:FasterXML/jackson-dataformat-smile.git</developerConnection>
+ <url>http://github.com/FasterXML/jackson-dataformat-smile</url>
+ <tag>jackson-dataformat-smile-2.2.2</tag>
+ </scm>
+
+ <properties>
+ <jackson.core.version>2.2.2</jackson.core.version>
+ <!-- Generate PackageVersion.java into this directory. -->
+ <packageVersion.dir>com/fasterxml/jackson/dataformat/smile</packageVersion.dir>
+ <packageVersion.package>${project.groupId}.smile</packageVersion.package>
+ <!--
+ | Configuration properties for the OSGi maven-bundle-plugin
+ -->
+ <osgi.export>${project.groupId}.smile.*;version=${project.version}
+</osgi.export>
+ <osgi.import>com.fasterxml.jackson.core
+,com.fasterxml.jackson.core.base
+,com.fasterxml.jackson.core.format
+,com.fasterxml.jackson.core.io
+,com.fasterxml.jackson.core.json
+,com.fasterxml.jackson.core.sym
+,com.fasterxml.jackson.core.util
+ </osgi.import>
+ </properties>
+
+ <dependencies>
+ <!-- Extends Jackson core -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.core.version}</version>
+ </dependency>
+
+ <!-- and for testing, JUnit is needed, as well as databind -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.core.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- Inherited from oss-base. Generate PackageVersion.java.-->
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <executions>
+ <execution>
+ <id>process-packageVersion</id>
+ <phase>generate-sources</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/release-notes/CREDITS b/release-notes/CREDITS
new file mode 100644
index 0000000..e526dbc
--- /dev/null
+++ b/release-notes/CREDITS
@@ -0,0 +1,11 @@
+Here are people who have contributed to Smile decoded plug-in development
+(version 2.0 onwards)
+(version numbers in brackets indicate release in which the problem was fixed)
+
+Tatu Saloranta, tatu.saloranta at iki.fi: author
+
+Steven Schlansker:
+
+ * Reported [Issue-2]: SmileParser failed to properly decoded surrogate-pair
+ characters for long strings
+ (2.0.2)
diff --git a/release-notes/VERSION b/release-notes/VERSION
new file mode 100644
index 0000000..b6c659a
--- /dev/null
+++ b/release-notes/VERSION
@@ -0,0 +1,96 @@
+Project: jackson-dataformat-smile
+Version: 2.2.2 (27-May-2013)
+
+No functional changes.
+
+------------------------------------------------------------------------
+=== History: ===
+------------------------------------------------------------------------
+
+2.2.1 (03-May-2013)
+
+- Fixed `SmileFactory.copy()` to retain configuration; also means that
+ underlying `JsonFactory` MUST be 2.2.1 or above (alas)
+
+2.2.0 (22-Apr-2013)
+
+New minor version, no functional changes.
+
+2.1.4 (26-Feb-2013)
+
+* Added checks in `SmileGenerator.writeBinary(InputStream)` to avoid infinite
+ looping (related to #55 of jackson-core)
+
+2.1.3 (31-Jan-2013)
+2.1.2 (04-Dec-2012)
+
+No functional changes.
+
+2.1.1 (11-Nov-2012)
+
+Fixes:
+
+* Do not throw an exception for empty content, even if header is
+ required; let parser implementation return null from 'nextToken()'
+
+2.1.0 (08-Oct-2012)
+ A minor 2.x release. The biggest additional new features are in streaming
+ processing, to support large binary data chunks without having to manually
+ create separate chunks.
+
+New features:
+
+* [Issue-3]: Implement `SmileGenerator.writeBinary()` variant that takes
+ InputStream, to allow incremental writes of large binary data
+* [Issue-4]: Implement `SmileParser.readBinary(...)` method to allow
+ incremental decoding of large binary payloads
+
+Other:
+
+* Override 'createParser()' and 'createGenerator()' methods for SmileFactory
+ to support [jackson-core/Issue-25]
+
+2.0.6 (05-Sep-2012)
+
+Fixes:
+
+* [Issue#7]: SmileParser.getCurrentName() "off-by-one" for START_OBJECT/ARRAY
+ (same as earlier [JACKSON-395])
+* [Issue#8]: Lost String index for `JsonGenerator` with resize, hash collision
+ (reported by M.T.Yurt)
+
+
+2.0.5 (23-Aug-2012)
+
+Fixes:
+
+* [Issue#6]: ObjectMapper.writeValue(File, ...) was not working due to
+ missing override in `SmileFactory`
+
+2.0.4 (26-May-2012)
+
+No fixes or changes, released along with other core components
+
+2.0.3:
+
+skipped -- only some modules used that version
+
+2.0.2 (14-May-2012)
+
+Fixes:
+
+* [Issue-2]: SmileParser failed to decode surrogate-pair characters for
+ long Strings
+ (reported by Steven S)
+
+2.0.1 (23-Apr-2012)
+
+No new features; re-compiled to require 2.0.1, to help JDK 1.5 compliancy.
+
+2.0.0 (25-Mar-2012)
+
+Fixes:
+
+ (all fixes up until 1.9.6)
+
+[entries for versions 1.x and earlier not retained; refer to earlier releases)
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/PackageVersion.java.in b/src/main/java/com/fasterxml/jackson/dataformat/smile/PackageVersion.java.in
new file mode 100644
index 0000000..7860aa1
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/PackageVersion.java.in
@@ -0,0 +1,20 @@
+package @package@;
+
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.core.Versioned;
+import com.fasterxml.jackson.core.util.VersionUtil;
+
+/**
+ * Automatically generated from PackageVersion.java.in during
+ * packageVersion-generate execution of maven-replacer-plugin in
+ * pom.xml.
+ */
+public final class PackageVersion implements Versioned {
+ public final static Version VERSION = VersionUtil.parseVersion(
+ "@projectversion@", "@projectgroupid@", "@projectartifactid@");
+
+ @Override
+ public Version version() {
+ return VERSION;
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileBufferRecycler.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileBufferRecycler.java
new file mode 100644
index 0000000..603bc09
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileBufferRecycler.java
@@ -0,0 +1,51 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+/**
+ * Simple helper class used for implementing simple reuse system for Smile-specific
+ * buffers that are used.
+ *
+ * @param <T> Type of name entries stored in arrays to recycle
+ */
+public class SmileBufferRecycler<T>
+{
+ public final static int DEFAULT_NAME_BUFFER_LENGTH = 64;
+
+ public final static int DEFAULT_STRING_VALUE_BUFFER_LENGTH = 64;
+
+ protected T[] _seenNamesBuffer;
+
+ protected T[] _seenStringValuesBuffer;
+
+ public SmileBufferRecycler() { }
+
+ public T[] allocSeenNamesBuffer()
+ {
+ // 11-Feb-2011, tatu: Used to alloc here; but due to generics, can't easily any more
+ T[] result = _seenNamesBuffer;
+ if (result != null) {
+ // let's ensure we don't retain it here, unless returned
+ _seenNamesBuffer = null;
+ // note: caller must have cleaned it up before returning
+ }
+ return result;
+ }
+
+ public T[] allocSeenStringValuesBuffer()
+ {
+ // 11-Feb-2011, tatu: Used to alloc here; but due to generics, can't easily any more
+ T[] result = _seenStringValuesBuffer;
+ if (result != null) {
+ _seenStringValuesBuffer = null;
+ // note: caller must have cleaned it up before returning
+ }
+ return result;
+ }
+
+ public void releaseSeenNamesBuffer(T[] buffer) {
+ _seenNamesBuffer = buffer;
+ }
+
+ public void releaseSeenStringValuesBuffer(T[] buffer) {
+ _seenStringValuesBuffer = buffer;
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileConstants.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileConstants.java
new file mode 100644
index 0000000..9451e6c
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileConstants.java
@@ -0,0 +1,367 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+/**
+ * Constants used by {@link SmileGenerator} and {@link SmileParser}
+ *
+ * @author tatu
+ */
+public final class SmileConstants
+{
+ /*
+ /**********************************************************
+ /* Thresholds
+ /**********************************************************
+ */
+
+ /**
+ * Encoding has special "short" forms for value Strings that can
+ * be represented by 64 bytes of UTF-8 or less.
+ */
+ public final static int MAX_SHORT_VALUE_STRING_BYTES = 64;
+
+ /**
+ * Encoding has special "short" forms for field names that can
+ * be represented by 64 bytes of UTF-8 or less.
+ */
+ public final static int MAX_SHORT_NAME_ASCII_BYTES = 64;
+
+ /**
+ * Maximum byte length for short non-ASCII names is slightly
+ * less due to having to reserve bytes 0xF8 and above (but
+ * we get one more as values 0 and 1 are not valid)
+ */
+ public final static int MAX_SHORT_NAME_UNICODE_BYTES = 56;
+
+ /**
+ * Longest back reference we use for field names is 10 bits; no point
+ * in keeping much more around
+ */
+ public final static int MAX_SHARED_NAMES = 1024;
+
+ /**
+ * Longest back reference we use for short shared String values is 10 bits,
+ * so up to (1 << 10) values to keep track of.
+ */
+ public final static int MAX_SHARED_STRING_VALUES = 1024;
+
+ /**
+ * Also: whereas we can refer to names of any length, we will only consider
+ * text values that are considered "tiny" or "short" (ones encoded with
+ * length prefix); this value thereby has to be maximum length of Strings
+ * that can be encoded as such.
+ */
+ public final static int MAX_SHARED_STRING_LENGTH_BYTES = 65;
+
+ /**
+ * And to make encoding logic tight and simple, we can always
+ * require that output buffer has this amount of space
+ * available before encoding possibly short String (3 bytes since
+ * longest UTF-8 encoded Java char is 3 bytes).
+ * Two extra bytes need to be reserved as well; first for token indicator,
+ * and second for terminating null byte (in case it's not a short String after all)
+ */
+ public final static int MIN_BUFFER_FOR_POSSIBLE_SHORT_STRING = 1 + (3 * 65);
+
+ /*
+ /**********************************************************
+ /* Byte markers
+ /**********************************************************
+ */
+
+ /**
+ * We need a byte marker to denote end of variable-length Strings. Although
+ * null byte is commonly used, let's try to avoid using it since it can't
+ * be embedded in Web Sockets content (similarly, 0xFF can't). There are
+ * multiple candidates for bytes UTF-8 can not have; 0xFC is chosen to
+ * allow reasonable ordering (highest values meaning most significant
+ * framing function; 0xFF being end-of-content and so on)
+ */
+ public final static int INT_MARKER_END_OF_STRING = 0xFC;
+
+ public final static byte BYTE_MARKER_END_OF_STRING = (byte) INT_MARKER_END_OF_STRING;
+
+ /**
+ * In addition we can use a marker to allow simple framing; splitting
+ * of physical data (like file) into distinct logical sections like
+ * JSON documents. 0xFF makes sense here since it is also used
+ * as end marker for Web Sockets.
+ */
+ public final static byte BYTE_MARKER_END_OF_CONTENT = (byte) 0xFF;
+
+ /*
+ /**********************************************************
+ /* Format header: put smile on your data...
+ /**********************************************************
+ */
+
+ /**
+ * First byte of data header (0x3A)
+ */
+ public final static byte HEADER_BYTE_1 = (byte) ':';
+
+ /**
+ * Second byte of data header (0x29)
+ */
+ public final static byte HEADER_BYTE_2 = (byte) ')';
+
+ /**
+ * Third byte of data header
+ */
+ public final static byte HEADER_BYTE_3 = (byte) '\n';
+
+ /**
+ * Current version consists of four zero bits (nibble)
+ */
+ public final static int HEADER_VERSION_0 = 0x0;
+
+ /**
+ * Fourth byte of data header; contains version nibble, may
+ * have flags
+ */
+ public final static byte HEADER_BYTE_4 = (HEADER_VERSION_0 << 4);
+
+ /**
+ * Indicator bit that indicates whether encoded content may
+ * have Shared names (back references to recently encoded field
+ * names). If no header available, must be
+ * processed as if this was set to true.
+ * If (and only if) header exists, and value is 0, can parser
+ * omit storing of seen names, as it is guaranteed that no back
+ * references exist.
+ */
+ public final static int HEADER_BIT_HAS_SHARED_NAMES = 0x01;
+
+ /**
+ * Indicator bit that indicates whether encoded content may
+ * have shared String values (back references to recently encoded
+ * 'short' String values, where short is defined as 64 bytes or less).
+ * If no header available, can be assumed to be 0 (false).
+ * If header exists, and bit value is 1, parsers has to store up
+ * to 1024 most recently seen distinct short String values.
+ */
+ public final static int HEADER_BIT_HAS_SHARED_STRING_VALUES = 0x02;
+
+ /**
+ * Indicator bit that indicates whether encoded content may
+ * contain raw (unquoted) binary values.
+ * If no header available, can be assumed to be 0 (false).
+ * If header exists, and bit value is 1, parser can not assume that
+ * specific byte values always have default meaning (specifically,
+ * content end marker 0xFF and header signature can be contained
+ * in binary values)
+ *<p>
+ * Note that this bit being true does not automatically mean that
+ * such raw binary content indeed exists; just that it may exist.
+ * This because header is written before any binary data may be
+ * written.
+ */
+ public final static int HEADER_BIT_HAS_RAW_BINARY = 0x04;
+
+ /*
+ /**********************************************************
+ /* Type prefixes: 3 MSB of token byte
+ /**********************************************************
+ */
+
+ public final static int TOKEN_PREFIX_INTEGER = 0x24;
+
+ public final static int TOKEN_PREFIX_FP = 0x28;
+
+ // Shared strings are back references for last 63 short (< 64 byte) string values
+ // NOTE: 0x00 is reserved, not used with current version (may be used in future)
+ public final static int TOKEN_PREFIX_SHARED_STRING_SHORT = 0x00;
+ // literals are put between 0x20 and 0x3F to reserve markers (smiley), along with ints/doubles
+ //public final static int TOKEN_PREFIX_MISC_NUMBERS = 0x20;
+
+ public final static int TOKEN_PREFIX_SHARED_STRING_LONG = 0xEC;
+
+ public final static int TOKEN_PREFIX_TINY_ASCII = 0x40;
+ public final static int TOKEN_PREFIX_SMALL_ASCII = 0x60;
+ public final static int TOKEN_PREFIX_TINY_UNICODE = 0x80;
+ public final static int TOKEN_PREFIX_SHORT_UNICODE = 0xA0;
+
+ // Small ints are 4-bit (-16 to +15) integer constants
+ public final static int TOKEN_PREFIX_SMALL_INT = 0xC0;
+
+ // And misc types have empty at the end too, to reserve 0xF8 - 0xFF
+ public final static int TOKEN_PREFIX_MISC_OTHER = 0xE0;
+
+ /*
+ /**********************************************************
+ /* Token literals, normal mode
+ /**********************************************************
+ */
+
+ // First, non-structured literals
+
+ public final static byte TOKEN_LITERAL_EMPTY_STRING = 0x20;
+ public final static byte TOKEN_LITERAL_NULL = 0x21;
+ public final static byte TOKEN_LITERAL_FALSE = 0x22;
+ public final static byte TOKEN_LITERAL_TRUE = 0x23;
+
+ // And then structured literals
+
+ public final static byte TOKEN_LITERAL_START_ARRAY = (byte) 0xF8;
+ public final static byte TOKEN_LITERAL_END_ARRAY = (byte) 0xF9;
+ public final static byte TOKEN_LITERAL_START_OBJECT = (byte) 0xFA;
+ public final static byte TOKEN_LITERAL_END_OBJECT = (byte) 0xFB;
+
+ /*
+ /**********************************************************
+ /* Subtype constants for misc text/binary types
+ /**********************************************************
+ */
+
+ /**
+ * @deprecated Since 2.1, use {@link #TOKEN_PREFIX_INTEGER} instead
+ */
+ @Deprecated
+ public final static int TOKEN_MISC_INTEGER = 0x24;
+
+ /**
+ * @deprecated Since 2.1, use {@link #TOKEN_PREFIX_FP} instead
+ */
+ @Deprecated
+ public final static int TOKEN_MISC_FP = 0x28;
+
+ /**
+ * Type (for misc, other) used for
+ * variable length UTF-8 encoded text, when it is known to only contain ASCII chars.
+ * Note: 2 LSB are reserved for future use; must be zeroes for now
+ */
+ public final static byte TOKEN_MISC_LONG_TEXT_ASCII = (byte) 0xE0;
+
+ /**
+ * Type (for misc, other) used
+ * for variable length UTF-8 encoded text, when it is NOT known to only contain ASCII chars
+ * (which means it MAY have multi-byte characters)
+ * Note: 2 LSB are reserved for future use; must be zeroes for now
+ */
+ public final static byte TOKEN_MISC_LONG_TEXT_UNICODE = (byte) 0xE4;
+
+ /**
+ * Type (for misc, other) used
+ * for "safe" (encoded by only using 7 LSB, giving 8/7 expansion ratio).
+ * This is usually done to ensure that certain bytes are never included
+ * in encoded data (like 0xFF)
+ * Note: 2 LSB are reserved for future use; must be zeroes for now
+ */
+ public final static byte TOKEN_MISC_BINARY_7BIT = (byte) 0xE8;
+
+ /**
+ * @deprecated (since 2.1) Use {@link #TOKEN_PREFIX_SHARED_STRING_LONG} instead
+ */
+ @Deprecated
+ public final static byte A_TOKEN_MISC_SHARED_STRING_LONG = (byte) 0xEC;
+
+ /**
+ * Raw binary data marker is specifically chosen as separate from
+ * other types, since it can have significant impact on framing
+ * (or rather fast scanning based on structure and framing markers).
+ */
+ public final static byte TOKEN_MISC_BINARY_RAW = (byte) 0xFD;
+
+ /*
+ /**********************************************************
+ /* Modifiers for numeric entries
+ /**********************************************************
+ */
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_INTEGER},
+ * indicating 32-bit integer (int)
+ */
+ public final static int TOKEN_MISC_INTEGER_32 = 0x00;
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_INTEGER},
+ * indicating 32-bit integer (long)
+ */
+ public final static int TOKEN_MISC_INTEGER_64 = 0x01;
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_INTEGER},
+ * indicating {@link java.math.BigInteger} type.
+ */
+ public final static int TOKEN_MISC_INTEGER_BIG = 0x02;
+
+ // Note: type 3 (0xF3) reserved for future use
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_FP},
+ * indicating 32-bit IEEE single precision floating point number.
+ */
+ public final static int TOKEN_MISC_FLOAT_32 = 0x00;
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_FP},
+ * indicating 64-bit IEEE double precision floating point number.
+ */
+ public final static int TOKEN_MISC_FLOAT_64 = 0x01;
+
+ /**
+ * Numeric subtype (2 LSB) for {@link #TOKEN_MISC_FP},
+ * indicating {@link java.math.BigDecimal} type.
+ */
+ public final static int TOKEN_MISC_FLOAT_BIG = 0x02;
+
+ // Note: type 3 (0xF7) reserved for future use
+
+ /*
+ /**********************************************************
+ /* Token types for keys
+ /**********************************************************
+ */
+
+ /**
+ * Let's use same code for empty key as for empty String value
+ */
+ public final static byte TOKEN_KEY_EMPTY_STRING = 0x20;
+
+ public final static int TOKEN_PREFIX_KEY_SHARED_LONG = 0x30;
+
+ public final static byte TOKEN_KEY_LONG_STRING = 0x34;
+
+ public final static int TOKEN_PREFIX_KEY_SHARED_SHORT = 0x40;
+
+ public final static int TOKEN_PREFIX_KEY_ASCII = 0x80;
+
+ public final static int TOKEN_PREFIX_KEY_UNICODE = 0xC0;
+
+ /*
+ /**********************************************************
+ /* Basic UTF-8 decode/encode table
+ /**********************************************************
+ */
+
+ /**
+ * Additionally we can combine UTF-8 decoding info into similar
+ * data table.
+ * Values indicate "byte length - 1"; meaning -1 is used for
+ * invalid bytes, 0 for single-byte codes, 1 for 2-byte codes
+ * and 2 for 3-byte codes.
+ */
+ public final static int[] sUtf8UnitLengths;
+ static {
+ int[] table = new int[256];
+ for (int c = 128; c < 256; ++c) {
+ int code;
+
+ // We'll add number of bytes needed for decoding
+ if ((c & 0xE0) == 0xC0) { // 2 bytes (0x0080 - 0x07FF)
+ code = 1;
+ } else if ((c & 0xF0) == 0xE0) { // 3 bytes (0x0800 - 0xFFFF)
+ code = 2;
+ } else if ((c & 0xF8) == 0xF0) {
+ // 4 bytes; double-char with surrogates and all...
+ code = 3;
+ } else {
+ // And -1 seems like a good "universal" error marker...
+ code = -1;
+ }
+ table[c] = code;
+ }
+ sUtf8UnitLengths = table;
+ }
+}
+
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileFactory.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileFactory.java
new file mode 100644
index 0000000..ad8e346
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileFactory.java
@@ -0,0 +1,564 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.net.URL;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.format.InputAccessor;
+import com.fasterxml.jackson.core.format.MatchStrength;
+import com.fasterxml.jackson.core.io.IOContext;
+
+/**
+ * Factory used for constructing {@link SmileParser} and {@link SmileGenerator}
+ * instances; both of which handle
+ * <a href="http://wiki.fasterxml.com/SmileFormat">Smile</a> encoded data.
+ *<p>
+ * Extends {@link JsonFactory} mostly so that users can actually use it in place
+ * of regular non-Smile factory instances.
+ *<p>
+ * Note on using non-byte-based sources/targets (char based, like
+ * {@link java.io.Reader} and {@link java.io.Writer}): these can not be
+ * used for Smile-format documents, and thus will either downgrade to
+ * textual JSON (when parsing), or throw exception (when trying to create
+ * generator).
+ *
+ * @author Tatu Saloranta
+ */
+public class SmileFactory extends JsonFactory
+{
+ private static final long serialVersionUID = -1696783009312472365L;
+
+ /*
+ /**********************************************************
+ /* Constants
+ /**********************************************************
+ */
+
+ /**
+ * Name used to identify Smile format.
+ * (and returned by {@link #getFormatName()}
+ */
+ public final static String FORMAT_NAME_SMILE = "Smile";
+
+ /**
+ * Bitfield (set of flags) of all parser features that are enabled
+ * by default.
+ */
+ final static int DEFAULT_SMILE_PARSER_FEATURE_FLAGS = SmileParser.Feature.collectDefaults();
+
+ /**
+ * Bitfield (set of flags) of all generator features that are enabled
+ * by default.
+ */
+ final static int DEFAULT_SMILE_GENERATOR_FEATURE_FLAGS = SmileGenerator.Feature.collectDefaults();
+
+ /*
+ /**********************************************************
+ /* Configuration
+ /**********************************************************
+ */
+
+ /**
+ * Whether non-supported methods (ones trying to output using
+ * char-based targets like {@link java.io.Writer}, for example)
+ * should be delegated to regular Jackson JSON processing
+ * (if set to true); or throw {@link UnsupportedOperationException}
+ * (if set to false)
+ */
+ protected boolean _cfgDelegateToTextual;
+
+ protected int _smileParserFeatures;
+ protected int _smileGeneratorFeatures;
+
+ /*
+ /**********************************************************
+ /* Factory construction, configuration
+ /**********************************************************
+ */
+
+ /**
+ * Default constructor used to create factory instances.
+ * Creation of a factory instance is a light-weight operation,
+ * but it is still a good idea to reuse limited number of
+ * factory instances (and quite often just a single instance):
+ * factories are used as context for storing some reused
+ * processing objects (such as symbol tables parsers use)
+ * and this reuse only works within context of a single
+ * factory instance.
+ */
+ public SmileFactory() { this(null); }
+
+ public SmileFactory(ObjectCodec oc) {
+ super(oc);
+ _smileParserFeatures = DEFAULT_SMILE_PARSER_FEATURE_FLAGS;
+ _smileGeneratorFeatures = DEFAULT_SMILE_GENERATOR_FEATURE_FLAGS;
+ }
+
+ /**
+ * Note: REQUIRES 2.2.1 -- unfortunate intra-patch dep but seems
+ * preferable to just leaving bug be as is
+ *
+ * @since 2.2.1
+ */
+ public SmileFactory(SmileFactory src, ObjectCodec oc)
+ {
+ super(src, oc);
+ _cfgDelegateToTextual = src._cfgDelegateToTextual;
+ _smileParserFeatures = src._smileParserFeatures;
+ _smileGeneratorFeatures = src._smileGeneratorFeatures;
+ }
+
+ // @since 2.1
+ @Override
+ public SmileFactory copy()
+ {
+ _checkInvalidCopy(SmileFactory.class);
+ // note: as with base class, must NOT copy mapper reference
+ return new SmileFactory(this, null);
+ }
+
+ public void delegateToTextual(boolean state) {
+ _cfgDelegateToTextual = state;
+ }
+
+ /*
+ /**********************************************************
+ /* Serializable overrides
+ /**********************************************************
+ */
+
+ /**
+ * Method that we need to override to actually make restoration go
+ * through constructors etc.
+ * Also: must be overridden by sub-classes as well.
+ */
+ @Override
+ protected Object readResolve() {
+ return new SmileFactory(this, _objectCodec);
+ }
+
+ /*
+ /**********************************************************
+ /* Versioned
+ /**********************************************************
+ */
+
+ @Override
+ public Version version() {
+ return PackageVersion.VERSION;
+ }
+
+ /*
+ /**********************************************************
+ /* Format detection functionality (since 1.8)
+ /**********************************************************
+ */
+
+ @Override
+ public String getFormatName() {
+ return FORMAT_NAME_SMILE;
+ }
+
+ // Defaults work fine for this:
+ // public boolean canUseSchema(FormatSchema schema) { }
+
+ /**
+ * Sub-classes need to override this method (as of 1.8)
+ */
+ @Override
+ public MatchStrength hasFormat(InputAccessor acc) throws IOException {
+ return SmileParserBootstrapper.hasSmileFormat(acc);
+ }
+
+ /*
+ /**********************************************************
+ /* Configuration, parser settings
+ /**********************************************************
+ */
+
+ /**
+ * Method for enabling or disabling specified parser feature
+ * (check {@link SmileParser.Feature} for list of features)
+ */
+ public final SmileFactory configure(SmileParser.Feature f, boolean state)
+ {
+ if (state) {
+ enable(f);
+ } else {
+ disable(f);
+ }
+ return this;
+ }
+
+ /**
+ * Method for enabling specified parser feature
+ * (check {@link SmileParser.Feature} for list of features)
+ */
+ public SmileFactory enable(SmileParser.Feature f) {
+ _smileParserFeatures |= f.getMask();
+ return this;
+ }
+
+ /**
+ * Method for disabling specified parser features
+ * (check {@link SmileParser.Feature} for list of features)
+ */
+ public SmileFactory disable(SmileParser.Feature f) {
+ _smileParserFeatures &= ~f.getMask();
+ return this;
+ }
+
+ /**
+ * Checked whether specified parser feature is enabled.
+ */
+ public final boolean isEnabled(SmileParser.Feature f) {
+ return (_smileParserFeatures & f.getMask()) != 0;
+ }
+
+ /*
+ /**********************************************************
+ /* Configuration, generator settings
+ /**********************************************************
+ */
+
+ /**
+ * Method for enabling or disabling specified generator feature
+ * (check {@link SmileGenerator.Feature} for list of features)
+ *
+ * @since 1.2
+ */
+ public final SmileFactory configure(SmileGenerator.Feature f, boolean state) {
+ if (state) {
+ enable(f);
+ } else {
+ disable(f);
+ }
+ return this;
+ }
+
+
+ /**
+ * Method for enabling specified generator features
+ * (check {@link SmileGenerator.Feature} for list of features)
+ */
+ public SmileFactory enable(SmileGenerator.Feature f) {
+ _smileGeneratorFeatures |= f.getMask();
+ return this;
+ }
+
+ /**
+ * Method for disabling specified generator feature
+ * (check {@link SmileGenerator.Feature} for list of features)
+ */
+ public SmileFactory disable(SmileGenerator.Feature f) {
+ _smileGeneratorFeatures &= ~f.getMask();
+ return this;
+ }
+
+ /**
+ * Check whether specified generator feature is enabled.
+ */
+ public final boolean isEnabled(SmileGenerator.Feature f) {
+ return (_smileGeneratorFeatures & f.getMask()) != 0;
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden parser factory methods, new (2.1)
+ /**********************************************************
+ */
+
+ @Override
+ public SmileParser createParser(File f)
+ throws IOException, JsonParseException
+ {
+ return _createParser(new FileInputStream(f), _createContext(f, true));
+ }
+
+ @Override
+ public SmileParser createParser(URL url)
+ throws IOException, JsonParseException
+ {
+ return _createParser(_optimizedStreamFromURL(url), _createContext(url, true));
+ }
+
+ @Override
+ public SmileParser createParser(InputStream in)
+ throws IOException, JsonParseException
+ {
+ return _createParser(in, _createContext(in, false));
+ }
+
+ //public JsonParser createJsonParser(Reader r)
+
+ @Override
+ public SmileParser createParser(byte[] data)
+ throws IOException, JsonParseException
+ {
+ IOContext ctxt = _createContext(data, true);
+ return _createParser(data, 0, data.length, ctxt);
+ }
+
+ @Override
+ public SmileParser createParser(byte[] data, int offset, int len)
+ throws IOException, JsonParseException
+ {
+ return _createParser(data, offset, len, _createContext(data, true));
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden parser factory methods, old (pre-2.1)
+ /**********************************************************
+ */
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createParser(File)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileParser createJsonParser(File f)
+ throws IOException, JsonParseException
+ {
+ return _createParser(new FileInputStream(f), _createContext(f, true));
+ }
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createParser(URL)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileParser createJsonParser(URL url)
+ throws IOException, JsonParseException
+ {
+ return _createParser(_optimizedStreamFromURL(url), _createContext(url, true));
+ }
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createParser(InputStream)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileParser createJsonParser(InputStream in)
+ throws IOException, JsonParseException
+ {
+ return _createParser(in, _createContext(in, false));
+ }
+
+ //public JsonParser createJsonParser(Reader r)
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createParser(byte[])} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileParser createJsonParser(byte[] data)
+ throws IOException, JsonParseException
+ {
+ IOContext ctxt = _createContext(data, true);
+ return _createParser(data, 0, data.length, ctxt);
+ }
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createParser(byte[],int,int)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileParser createJsonParser(byte[] data, int offset, int len)
+ throws IOException, JsonParseException
+ {
+ return _createParser(data, offset, len, _createContext(data, true));
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden generator factory methods, new (2.1)
+ /**********************************************************
+ */
+
+ /**
+ * Method for constructing {@link JsonGenerator} for generating
+ * Smile-encoded output.
+ *<p>
+ * Since Smile format always uses UTF-8 internally, <code>enc</code>
+ * argument is ignored.
+ */
+ @Override
+ public SmileGenerator createGenerator(OutputStream out, JsonEncoding enc)
+ throws IOException
+ {
+ // false -> we won't manage the stream unless explicitly directed to
+ return _createGenerator(out, _createContext(out, false));
+ }
+
+ /**
+ * Method for constructing {@link JsonGenerator} for generating
+ * Smile-encoded output.
+ *<p>
+ * Since Smile format always uses UTF-8 internally, no encoding need
+ * to be passed to this method.
+ */
+ @Override
+ public SmileGenerator createGenerator(OutputStream out) throws IOException
+ {
+ // false -> we won't manage the stream unless explicitly directed to
+ return _createGenerator(out, _createContext(out, false));
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden generator factory methods, old (pre-2.1)
+ /**********************************************************
+ */
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createGenerator(OutputStream)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileGenerator createJsonGenerator(OutputStream out, JsonEncoding enc)
+ throws IOException
+ {
+ // false -> we won't manage the stream unless explicitly directed to
+ return _createGenerator(out, _createContext(out, false));
+ }
+
+ /**
+ * @deprecated Since 2.1 Use {@link #createGenerator(OutputStream)} instead
+ * @since 2.1
+ */
+ @Deprecated
+ @Override
+ public SmileGenerator createJsonGenerator(OutputStream out) throws IOException
+ {
+ // false -> we won't manage the stream unless explicitly directed to
+ IOContext ctxt = _createContext(out, false);
+ return _createGenerator(out, ctxt);
+ }
+
+ @Deprecated
+ @Override
+ protected SmileGenerator _createUTF8JsonGenerator(OutputStream out, IOContext ctxt)
+ throws IOException
+ {
+ return _createGenerator(out, ctxt);
+ }
+
+ /*
+ /******************************************************
+ /* Overridden internal factory methods
+ /******************************************************
+ */
+
+ //protected IOContext _createContext(Object srcRef, boolean resourceManaged)
+
+ /**
+ * Overridable factory method that actually instantiates desired
+ * parser.
+ */
+ @Override
+ protected SmileParser _createParser(InputStream in, IOContext ctxt)
+ throws IOException, JsonParseException
+ {
+ return new SmileParserBootstrapper(ctxt, in).constructParser(_parserFeatures,
+ _smileParserFeatures, isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES),
+ _objectCodec, _rootByteSymbols);
+ }
+
+ /**
+ * Overridable factory method that actually instantiates desired
+ * parser.
+ */
+ @Override
+ protected JsonParser _createParser(Reader r, IOContext ctxt)
+ throws IOException, JsonParseException
+ {
+ if (_cfgDelegateToTextual) {
+ return super._createParser(r, ctxt);
+ }
+ throw new UnsupportedOperationException("Can not create generator for non-byte-based target");
+ }
+
+ /**
+ * Overridable factory method that actually instantiates desired
+ * parser.
+ */
+ @Override
+ protected SmileParser _createParser(byte[] data, int offset, int len, IOContext ctxt)
+ throws IOException, JsonParseException
+ {
+ return new SmileParserBootstrapper(ctxt, data, offset, len).constructParser(
+ _parserFeatures, _smileParserFeatures,
+ isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES),
+ _objectCodec, _rootByteSymbols);
+ }
+
+ /**
+ * Overridable factory method that actually instantiates desired
+ * generator.
+ */
+ @Override
+ protected JsonGenerator _createGenerator(Writer out, IOContext ctxt)
+ throws IOException
+ {
+ if (_cfgDelegateToTextual) {
+ return super._createGenerator(out, ctxt);
+ }
+ throw new UnsupportedOperationException("Can not create generator for non-byte-based target");
+ }
+
+ @Override
+ protected JsonGenerator _createUTF8Generator(OutputStream out, IOContext ctxt) throws IOException {
+ return _createGenerator(out, ctxt);
+ }
+
+ //public BufferRecycler _getBufferRecycler()
+
+ @Override
+ protected Writer _createWriter(OutputStream out, JsonEncoding enc, IOContext ctxt) throws IOException
+ {
+ if (_cfgDelegateToTextual) {
+ return super._createWriter(out, enc, ctxt);
+ }
+ throw new UnsupportedOperationException("Can not create generator for non-byte-based target");
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods
+ /**********************************************************
+ */
+
+ protected SmileGenerator _createGenerator(OutputStream out, IOContext ctxt)
+ throws IOException
+ {
+ int feats = _smileGeneratorFeatures;
+ /* One sanity check: MUST write header if shared string values setting is enabled,
+ * or quoting of binary data disabled.
+ * But should we force writing, or throw exception, if settings are in conflict?
+ * For now, let's error out...
+ */
+ SmileGenerator gen = new SmileGenerator(ctxt, _generatorFeatures, feats, _objectCodec, out);
+ if ((feats & SmileGenerator.Feature.WRITE_HEADER.getMask()) != 0) {
+ gen.writeHeader();
+ } else {
+ if ((feats & SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES.getMask()) != 0) {
+ throw new JsonGenerationException(
+ "Inconsistent settings: WRITE_HEADER disabled, but CHECK_SHARED_STRING_VALUES enabled; can not construct generator"
+ +" due to possible data loss (either enable WRITE_HEADER, or disable CHECK_SHARED_STRING_VALUES to resolve)");
+ }
+ if ((feats & SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT.getMask()) == 0) {
+ throw new JsonGenerationException(
+ "Inconsistent settings: WRITE_HEADER disabled, but ENCODE_BINARY_AS_7BIT disabled; can not construct generator"
+ +" due to possible data loss (either enable WRITE_HEADER, or ENCODE_BINARY_AS_7BIT to resolve)");
+ }
+ }
+ return gen;
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileGenerator.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileGenerator.java
new file mode 100644
index 0000000..06d9787
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileGenerator.java
@@ -0,0 +1,2298 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.lang.ref.SoftReference;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.io.*;
+import com.fasterxml.jackson.core.json.JsonWriteContext;
+import com.fasterxml.jackson.core.base.GeneratorBase;
+
+import static com.fasterxml.jackson.dataformat.smile.SmileConstants.*;
+
+/**
+ * {@link JsonGenerator} implementation for the experimental "Binary JSON Infoset".
+ *
+ * @author tatu
+ */
+public class SmileGenerator
+ extends GeneratorBase
+{
+ /**
+ * Enumeration that defines all togglable features for Smile generators.
+ */
+ public enum Feature {
+ /**
+ * Whether to write 4-byte header sequence when starting output or not.
+ * If disabled, no header is written; this may be useful in embedded cases
+ * where context is enough to know that content is encoded using this format.
+ * Note, however, that omitting header means that default settings for
+ * shared names/string values can not be changed.
+ *<p>
+ * Default setting is true, meaning that header will be written.
+ */
+ WRITE_HEADER(true),
+
+ /**
+ * Whether write byte marker that signifies end of logical content segment
+ * ({@link SmileConstants#BYTE_MARKER_END_OF_CONTENT}) when
+ * {@link #close} is called or not. This can be useful when outputting
+ * multiple adjacent logical content segments (documents) into single
+ * physical output unit (file).
+ *<p>
+ * Default setting is false meaning that such marker is not written.
+ */
+ WRITE_END_MARKER(false),
+
+ /**
+ * Whether to use simple 7-bit per byte encoding for binary content when output.
+ * This is necessary ensure that byte 0xFF will never be included in content output.
+ * For other data types this limitation is handled automatically; but since overhead
+ * for binary data (14% size expansion, processing overhead) is non-negligible,
+ * it is not enabled by default. If no binary data is output, feature has no effect.
+ *<p>
+ * Default setting is true, indicating that binary data is quoted as 7-bit bytes
+ * instead of written raw.
+ */
+ ENCODE_BINARY_AS_7BIT(true),
+
+ /**
+ * Whether generator should check if it can "share" field names during generating
+ * content or not. If enabled, can replace repeating field names with back references,
+ * which are more compact and should faster to decode. Downside is that there is some
+ * overhead for writing (need to track existing values, check), as well as decoding.
+ *<p>
+ * Since field names tend to repeat quite often, this setting is enabled by default.
+ */
+ CHECK_SHARED_NAMES(true),
+
+ /**
+ * Whether generator should check if it can "share" short (at most 64 bytes encoded)
+ * String value during generating
+ * content or not. If enabled, can replace repeating Short String values with back references,
+ * which are more compact and should faster to decode. Downside is that there is some
+ * overhead for writing (need to track existing values, check), as well as decoding.
+ *<p>
+ * Since efficiency of this option depends a lot on type of content being produced,
+ * this option is disabled by default, and should only be enabled if it is likely that
+ * same values repeat relatively often.
+ */
+ CHECK_SHARED_STRING_VALUES(false)
+ ;
+
+ protected final boolean _defaultState;
+ protected final int _mask;
+
+ /**
+ * Method that calculates bit set (flags) of all features that
+ * are enabled by default.
+ */
+ public static int collectDefaults()
+ {
+ int flags = 0;
+ for (Feature f : values()) {
+ if (f.enabledByDefault()) {
+ flags |= f.getMask();
+ }
+ }
+ return flags;
+ }
+
+ private Feature(boolean defaultState) {
+ _defaultState = defaultState;
+ _mask = (1 << ordinal());
+ }
+
+ public boolean enabledByDefault() { return _defaultState; }
+ public int getMask() { return _mask; }
+ }
+
+ /**
+ * Helper class used for keeping track of possibly shareable String
+ * references (for field names and/or short String values)
+ */
+ protected final static class SharedStringNode
+ {
+ public final String value;
+ public final int index;
+ public SharedStringNode next;
+
+ public SharedStringNode(String value, int index, SharedStringNode next)
+ {
+ this.value = value;
+ this.index = index;
+ this.next = next;
+ }
+ }
+
+ /**
+ * To simplify certain operations, we require output buffer length
+ * to allow outputting of contiguous 256 character UTF-8 encoded String
+ * value. Length of the longest UTF-8 code point (from Java char) is 3 bytes,
+ * and we need both initial token byte and single-byte end marker
+ * so we get following value.
+ *<p>
+ * Note: actually we could live with shorter one; absolute minimum would
+ * be for encoding 64-character Strings.
+ */
+ private final static int MIN_BUFFER_LENGTH = (3 * 256) + 2;
+
+ protected final static byte TOKEN_BYTE_LONG_STRING_ASCII = TOKEN_MISC_LONG_TEXT_ASCII;
+
+ protected final static byte TOKEN_BYTE_INT_32 = (byte) (SmileConstants.TOKEN_PREFIX_INTEGER + TOKEN_MISC_INTEGER_32);
+ protected final static byte TOKEN_BYTE_INT_64 = (byte) (SmileConstants.TOKEN_PREFIX_INTEGER + TOKEN_MISC_INTEGER_64);
+ protected final static byte TOKEN_BYTE_BIG_INTEGER = (byte) (SmileConstants.TOKEN_PREFIX_INTEGER + TOKEN_MISC_INTEGER_BIG);
+
+ protected final static byte TOKEN_BYTE_FLOAT_32 = (byte) (SmileConstants.TOKEN_PREFIX_FP | TOKEN_MISC_FLOAT_32);
+ protected final static byte TOKEN_BYTE_FLOAT_64 = (byte) (SmileConstants.TOKEN_PREFIX_FP | TOKEN_MISC_FLOAT_64);
+ protected final static byte TOKEN_BYTE_BIG_DECIMAL = (byte) (SmileConstants.TOKEN_PREFIX_FP | TOKEN_MISC_FLOAT_BIG);
+
+ protected final static int SURR1_FIRST = 0xD800;
+ protected final static int SURR1_LAST = 0xDBFF;
+ protected final static int SURR2_FIRST = 0xDC00;
+ protected final static int SURR2_LAST = 0xDFFF;
+
+ protected final static long MIN_INT_AS_LONG = (long) Integer.MIN_VALUE;
+ protected final static long MAX_INT_AS_LONG = (long) Integer.MAX_VALUE;
+
+ /*
+ /**********************************************************
+ /* Configuration
+ /**********************************************************
+ */
+
+ final protected IOContext _ioContext;
+
+ final protected OutputStream _out;
+
+ /**
+ * Bit flag composed of bits that indicate which
+ * {@link com.fasterxml.jackson.dataformat.smile.SmileGenerator.Feature}s
+ * are enabled.
+ */
+ protected int _smileFeatures;
+
+ /**
+ * Helper object used for low-level recycling of Smile-generator
+ * specific buffers.
+ */
+ final protected SmileBufferRecycler<SharedStringNode> _smileBufferRecycler;
+
+ /*
+ /**********************************************************
+ /* Output buffering
+ /**********************************************************
+ */
+
+ /**
+ * Intermediate buffer in which contents are buffered before
+ * being written using {@link #_out}.
+ */
+ protected byte[] _outputBuffer;
+
+ /**
+ * Pointer to the next available byte in {@link #_outputBuffer}
+ */
+ protected int _outputTail = 0;
+
+ /**
+ * Offset to index after the last valid index in {@link #_outputBuffer}.
+ * Typically same as length of the buffer.
+ */
+ protected final int _outputEnd;
+
+ /**
+ * Intermediate buffer in which characters of a String are copied
+ * before being encoded.
+ */
+ protected char[] _charBuffer;
+
+ protected final int _charBufferLength;
+
+ /**
+ * Let's keep track of how many bytes have been output, may prove useful
+ * when debugging. This does <b>not</b> include bytes buffered in
+ * the output buffer, just bytes that have been written using underlying
+ * stream writer.
+ */
+ protected int _bytesWritten;
+
+ /*
+ /**********************************************************
+ /* Shared String detection
+ /**********************************************************
+ */
+
+ /**
+ * Raw data structure used for checking whether field name to
+ * write can be output using back reference or not.
+ */
+ protected SharedStringNode[] _seenNames;
+
+ /**
+ * Number of entries in {@link #_seenNames}; -1 if no shared name
+ * detection is enabled
+ */
+ protected int _seenNameCount;
+
+ /**
+ * Raw data structure used for checking whether String value to
+ * write can be output using back reference or not.
+ */
+ protected SharedStringNode[] _seenStringValues;
+
+ /**
+ * Number of entries in {@link #_seenStringValues}; -1 if no shared text value
+ * detection is enabled
+ */
+ protected int _seenStringValueCount;
+
+ /**
+ * Flag that indicates whether the output buffer is recycable (and
+ * needs to be returned to recycler once we are done) or not.
+ */
+ protected boolean _bufferRecyclable;
+
+ /*
+ /**********************************************************
+ /* Thread-local recycling
+ /**********************************************************
+ */
+
+ /**
+ * This <code>ThreadLocal</code> contains a {@link java.lang.ref.SoftReference}
+ * to a buffer recycler used to provide a low-cost
+ * buffer recycling for Smile-specific buffers.
+ */
+ final protected static ThreadLocal<SoftReference<SmileBufferRecycler<SharedStringNode>>> _smileRecyclerRef
+ = new ThreadLocal<SoftReference<SmileBufferRecycler<SharedStringNode>>>();
+
+ /*
+ /**********************************************************
+ /* Life-cycle
+ /**********************************************************
+ */
+
+ public SmileGenerator(IOContext ctxt, int jsonFeatures, int smileFeatures,
+ ObjectCodec codec, OutputStream out)
+ {
+ super(jsonFeatures, codec);
+ _smileFeatures = smileFeatures;
+ _ioContext = ctxt;
+ _smileBufferRecycler = _smileBufferRecycler();
+ _out = out;
+ _bufferRecyclable = true;
+ _outputBuffer = ctxt.allocWriteEncodingBuffer();
+ _outputEnd = _outputBuffer.length;
+ _charBuffer = ctxt.allocConcatBuffer();
+ _charBufferLength = _charBuffer.length;
+ // let's just sanity check to prevent nasty odd errors
+ if (_outputEnd < MIN_BUFFER_LENGTH) {
+ throw new IllegalStateException("Internal encoding buffer length ("+_outputEnd
+ +") too short, must be at least "+MIN_BUFFER_LENGTH);
+ }
+ if ((smileFeatures & Feature.CHECK_SHARED_NAMES.getMask()) == 0) {
+ _seenNames = null;
+ _seenNameCount = -1;
+ } else {
+ _seenNames = _smileBufferRecycler.allocSeenNamesBuffer();
+ if (_seenNames == null) {
+ _seenNames = new SharedStringNode[SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH];
+ }
+ _seenNameCount = 0;
+ }
+
+ if ((smileFeatures & Feature.CHECK_SHARED_STRING_VALUES.getMask()) == 0) {
+ _seenStringValues = null;
+ _seenStringValueCount = -1;
+ } else {
+ _seenStringValues = _smileBufferRecycler.allocSeenStringValuesBuffer();
+ if (_seenStringValues == null) {
+ _seenStringValues = new SharedStringNode[SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH];
+ }
+ _seenStringValueCount = 0;
+ }
+}
+
+ public SmileGenerator(IOContext ctxt, int jsonFeatures, int smileFeatures,
+ ObjectCodec codec, OutputStream out, byte[] outputBuffer, int offset, boolean bufferRecyclable)
+ {
+ super(jsonFeatures, codec);
+ _smileFeatures = smileFeatures;
+ _ioContext = ctxt;
+ _smileBufferRecycler = _smileBufferRecycler();
+ _out = out;
+ _bufferRecyclable = bufferRecyclable;
+ _outputTail = offset;
+ _outputBuffer = outputBuffer;
+ _outputEnd = _outputBuffer.length;
+ _charBuffer = ctxt.allocConcatBuffer();
+ _charBufferLength = _charBuffer.length;
+ // let's just sanity check to prevent nasty odd errors
+ if (_outputEnd < MIN_BUFFER_LENGTH) {
+ throw new IllegalStateException("Internal encoding buffer length ("+_outputEnd
+ +") too short, must be at least "+MIN_BUFFER_LENGTH);
+ }
+ if ((smileFeatures & Feature.CHECK_SHARED_NAMES.getMask()) == 0) {
+ _seenNames = null;
+ _seenNameCount = -1;
+ } else {
+ _seenNames = _smileBufferRecycler.allocSeenNamesBuffer();
+ if (_seenNames == null) {
+ _seenNames = new SharedStringNode[SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH];
+ }
+ _seenNameCount = 0;
+ }
+
+ if ((smileFeatures & Feature.CHECK_SHARED_STRING_VALUES.getMask()) == 0) {
+ _seenStringValues = null;
+ _seenStringValueCount = -1;
+ } else {
+ _seenStringValues = _smileBufferRecycler.allocSeenStringValuesBuffer();
+ if (_seenStringValues == null) {
+ _seenStringValues = new SharedStringNode[SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH];
+ }
+ _seenStringValueCount = 0;
+ }
+ }
+
+ /**
+ * Method that can be called to explicitly write Smile document header.
+ * Note that usually you do not need to call this for first document to output,
+ * but rather only if you intend to write multiple root-level documents
+ * with same generator (and even in that case this is optional thing to do).
+ * As a result usually only {@link SmileFactory} calls this method.
+ */
+ public void writeHeader() throws IOException
+ {
+ int last = HEADER_BYTE_4;
+ if ((_smileFeatures & Feature.CHECK_SHARED_NAMES.getMask()) != 0) {
+ last |= SmileConstants.HEADER_BIT_HAS_SHARED_NAMES;
+ }
+ if ((_smileFeatures & Feature.CHECK_SHARED_STRING_VALUES.getMask()) != 0) {
+ last |= SmileConstants.HEADER_BIT_HAS_SHARED_STRING_VALUES;
+ }
+ if ((_smileFeatures & Feature.ENCODE_BINARY_AS_7BIT.getMask()) == 0) {
+ last |= SmileConstants.HEADER_BIT_HAS_RAW_BINARY;
+ }
+ _writeBytes(HEADER_BYTE_1, HEADER_BYTE_2, HEADER_BYTE_3, (byte) last);
+ }
+
+ protected final static SmileBufferRecycler<SharedStringNode> _smileBufferRecycler()
+ {
+ SoftReference<SmileBufferRecycler<SharedStringNode>> ref = _smileRecyclerRef.get();
+ SmileBufferRecycler<SharedStringNode> br = (ref == null) ? null : ref.get();
+
+ if (br == null) {
+ br = new SmileBufferRecycler<SharedStringNode>();
+ _smileRecyclerRef.set(new SoftReference<SmileBufferRecycler<SharedStringNode>>(br));
+ }
+ return br;
+ }
+
+ /*
+ /**********************************************************
+ /* Versioned
+ /**********************************************************
+ */
+
+ @Override
+ public Version version() {
+ return PackageVersion.VERSION;
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden methods, configuration
+ /**********************************************************
+ */
+
+ /**
+ * No way (or need) to indent anything, so let's block any attempts.
+ * (should we throw an exception instead?)
+ */
+ @Override
+ public JsonGenerator useDefaultPrettyPrinter()
+ {
+ return this;
+ }
+
+ /**
+ * No way (or need) to indent anything, so let's block any attempts.
+ * (should we throw an exception instead?)
+ */
+ @Override
+ public JsonGenerator setPrettyPrinter(PrettyPrinter pp) {
+ return this;
+ }
+
+ @Override
+ public Object getOutputTarget() {
+ return _out;
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden methods, write methods
+ /**********************************************************
+ */
+
+ /* And then methods overridden to make final, streamline some
+ * aspects...
+ */
+
+ @Override
+ public final void writeFieldName(String name) throws IOException, JsonGenerationException
+ {
+ if (_writeContext.writeFieldName(name) == JsonWriteContext.STATUS_EXPECT_VALUE) {
+ _reportError("Can not write a field name, expecting a value");
+ }
+ _writeFieldName(name);
+ }
+
+ @Override
+ public final void writeFieldName(SerializableString name)
+ throws IOException, JsonGenerationException
+ {
+ // Object is a value, need to verify it's allowed
+ if (_writeContext.writeFieldName(name.getValue()) == JsonWriteContext.STATUS_EXPECT_VALUE) {
+ _reportError("Can not write a field name, expecting a value");
+ }
+ _writeFieldName(name);
+ }
+
+ @Override
+ public final void writeStringField(String fieldName, String value)
+ throws IOException, JsonGenerationException
+ {
+ if (_writeContext.writeFieldName(fieldName) == JsonWriteContext.STATUS_EXPECT_VALUE) {
+ _reportError("Can not write a field name, expecting a value");
+ }
+ _writeFieldName(fieldName);
+ writeString(value);
+ }
+
+ /*
+ /**********************************************************
+ /* Extended API, configuration
+ /**********************************************************
+ */
+
+ public SmileGenerator enable(Feature f) {
+ _smileFeatures |= f.getMask();
+ return this;
+ }
+
+ public SmileGenerator disable(Feature f) {
+ _smileFeatures &= ~f.getMask();
+ return this;
+ }
+
+ public final boolean isEnabled(Feature f) {
+ return (_smileFeatures & f.getMask()) != 0;
+ }
+
+ public SmileGenerator configure(Feature f, boolean state) {
+ if (state) {
+ enable(f);
+ } else {
+ disable(f);
+ }
+ return this;
+ }
+
+ /*
+ /**********************************************************
+ /* Extended API, other
+ /**********************************************************
+ */
+
+ /**
+ * Method for directly inserting specified byte in output at
+ * current position.
+ *<p>
+ * NOTE: only use this method if you really know what you are doing.
+ */
+ public void writeRaw(byte b) throws IOException, JsonGenerationException
+ {
+ _writeByte(TOKEN_LITERAL_START_ARRAY);
+ }
+
+ /**
+ * Method for directly inserting specified bytes in output at
+ * current position.
+ *<p>
+ * NOTE: only use this method if you really know what you are doing.
+ */
+ public void writeBytes(byte[] data, int offset, int len) throws IOException
+ {
+ _writeBytes(data, offset, len);
+ }
+
+ /*
+ /**********************************************************
+ /* Output method implementations, structural
+ /**********************************************************
+ */
+
+ @Override
+ public final void writeStartArray() throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("start an array");
+ _writeContext = _writeContext.createChildArrayContext();
+ _writeByte(TOKEN_LITERAL_START_ARRAY);
+ }
+
+ @Override
+ public final void writeEndArray() throws IOException, JsonGenerationException
+ {
+ if (!_writeContext.inArray()) {
+ _reportError("Current context not an ARRAY but "+_writeContext.getTypeDesc());
+ }
+ _writeByte(TOKEN_LITERAL_END_ARRAY);
+ _writeContext = _writeContext.getParent();
+ }
+
+ @Override
+ public final void writeStartObject() throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("start an object");
+ _writeContext = _writeContext.createChildObjectContext();
+ _writeByte(TOKEN_LITERAL_START_OBJECT);
+ }
+
+ @Override
+ public final void writeEndObject() throws IOException, JsonGenerationException
+ {
+ if (!_writeContext.inObject()) {
+ _reportError("Current context not an object but "+_writeContext.getTypeDesc());
+ }
+ _writeContext = _writeContext.getParent();
+ _writeByte(TOKEN_LITERAL_END_OBJECT);
+ }
+
+ private final void _writeFieldName(String name)
+ throws IOException, JsonGenerationException
+ {
+ int len = name.length();
+ if (len == 0) {
+ _writeByte(TOKEN_KEY_EMPTY_STRING);
+ return;
+ }
+ // First: is it something we can share?
+ if (_seenNameCount >= 0) {
+ int ix = _findSeenName(name);
+ if (ix >= 0) {
+ _writeSharedNameReference(ix);
+ return;
+ }
+ }
+ if (len > MAX_SHORT_NAME_UNICODE_BYTES) { // can not be a 'short' String; off-line (rare case)
+ _writeNonShortFieldName(name, len);
+ return;
+ }
+
+ // first: ensure we have enough space
+ if ((_outputTail + MIN_BUFFER_FOR_POSSIBLE_SHORT_STRING) >= _outputEnd) {
+ _flushBuffer();
+ }
+ // then let's copy String chars to char buffer, faster than using getChar (measured, profiled)
+ name.getChars(0, len, _charBuffer, 0);
+ int origOffset = _outputTail;
+ ++_outputTail; // to reserve space for type token
+ int byteLen = _shortUTF8Encode(_charBuffer, 0, len);
+ byte typeToken;
+
+ // ASCII?
+ if (byteLen == len) {
+ if (byteLen <= MAX_SHORT_NAME_ASCII_BYTES) { // yes, is short indeed
+ typeToken = (byte) ((TOKEN_PREFIX_KEY_ASCII - 1) + byteLen);
+ } else { // longer albeit ASCII
+ typeToken = TOKEN_KEY_LONG_STRING;
+ // and we will need String end marker byte
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+ } else { // not all ASCII
+ if (byteLen <= MAX_SHORT_NAME_UNICODE_BYTES) { // yes, is short indeed
+ // note: since 2 is smaller allowed length, offset differs from one used for
+ typeToken = (byte) ((TOKEN_PREFIX_KEY_UNICODE - 2) + byteLen);
+ } else { // nope, longer non-ASCII Strings
+ typeToken = TOKEN_KEY_LONG_STRING;
+ // and we will need String end marker byte
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+ }
+ // and then sneak in type token now that know the details
+ _outputBuffer[origOffset] = typeToken;
+ // Also, keep track if we can use back-references (shared names)
+ if (_seenNameCount >= 0) {
+ _addSeenName(name);
+ }
+ }
+
+ private final void _writeNonShortFieldName(final String name, final int len)
+ throws IOException, JsonGenerationException
+ {
+ _writeByte(TOKEN_KEY_LONG_STRING);
+ // can we still make a temp copy?
+ if (len > _charBufferLength) { // nah, not even that
+ _slowUTF8Encode(name);
+ } else { // yep.
+ name.getChars(0, len, _charBuffer, 0);
+ // but will encoded version fit in buffer?
+ int maxLen = len + len + len;
+ if (maxLen <= _outputBuffer.length) { // yes indeed
+ if ((_outputTail + maxLen) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _shortUTF8Encode(_charBuffer, 0, len);
+ } else { // nope, need bit slower variant
+ _mediumUTF8Encode(_charBuffer, 0, len);
+ }
+ }
+ if (_seenNameCount >= 0) {
+ _addSeenName(name);
+ }
+ if (_outputTail >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+
+ protected final void _writeFieldName(SerializableString name)
+ throws IOException, JsonGenerationException
+ {
+ final int charLen = name.charLength();
+ if (charLen == 0) {
+ _writeByte(TOKEN_KEY_EMPTY_STRING);
+ return;
+ }
+ // Then: is it something we can share?
+ if (_seenNameCount >= 0) {
+ int ix = _findSeenName(name.getValue());
+ if (ix >= 0) {
+ _writeSharedNameReference(ix);
+ return;
+ }
+ }
+ final byte[] bytes = name.asUnquotedUTF8();
+ final int byteLen = bytes.length;
+ if (byteLen != charLen) {
+ _writeFieldNameUnicode(name, bytes);
+ return;
+ }
+ // Common case: short ASCII name that fits in buffer as is
+ if (byteLen <= MAX_SHORT_NAME_ASCII_BYTES) {
+ // output buffer is bigger than what we need, always, so
+ if ((_outputTail + byteLen) >= _outputEnd) { // need marker byte and actual bytes
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = (byte) ((TOKEN_PREFIX_KEY_ASCII - 1) + byteLen);
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ } else {
+ _writeLongAsciiFieldName(bytes);
+ }
+ // Also, keep track if we can use back-references (shared names)
+ if (_seenNameCount >= 0) {
+ _addSeenName(name.getValue());
+ }
+ }
+
+ private final void _writeLongAsciiFieldName(byte[] bytes)
+ throws IOException, JsonGenerationException
+ {
+ final int byteLen = bytes.length;
+ if (_outputTail >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = TOKEN_KEY_LONG_STRING;
+ // Ok. Enough room?
+ if ((_outputTail + byteLen + 1) < _outputEnd) {
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ } else {
+ _flushBuffer();
+ // either way, do intermediate copy if name is relatively short
+ // Need to copy?
+ if (byteLen < MIN_BUFFER_LENGTH) {
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ } else {
+ // otherwise, just write as is
+ if (_outputTail > 0) {
+ _flushBuffer();
+ }
+ _out.write(bytes, 0, byteLen);
+ }
+ }
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+
+ protected final void _writeFieldNameUnicode(SerializableString name, byte[] bytes)
+ throws IOException, JsonGenerationException
+ {
+ final int byteLen = bytes.length;
+
+ // Common case: short Unicode name that fits in output buffer
+ if (byteLen <= MAX_SHORT_NAME_UNICODE_BYTES) {
+ if ((_outputTail + byteLen) >= _outputEnd) { // need marker byte and actual bytes
+ _flushBuffer();
+ }
+ // note: since 2 is smaller allowed length, offset differs from one used for
+ _outputBuffer[_outputTail++] = (byte) ((TOKEN_PREFIX_KEY_UNICODE - 2) + byteLen);
+
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ // Also, keep track if we can use back-references (shared names)
+ if (_seenNameCount >= 0) {
+ _addSeenName(name.getValue());
+ }
+ return;
+ }
+ if (_outputTail >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = TOKEN_KEY_LONG_STRING;
+ // Ok. Enough room?
+ if ((_outputTail + byteLen + 1) < _outputEnd) {
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ } else {
+ _flushBuffer();
+ // either way, do intermediate copy if name is relatively short
+ // Need to copy?
+ if (byteLen < MIN_BUFFER_LENGTH) {
+ System.arraycopy(bytes, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ } else {
+ // otherwise, just write as is
+ if (_outputTail > 0) {
+ _flushBuffer();
+ }
+ _out.write(bytes, 0, byteLen);
+ }
+ }
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ // Also, keep track if we can use back-references (shared names)
+ if (_seenNameCount >= 0) {
+ _addSeenName(name.getValue());
+ }
+ }
+
+ private final void _writeSharedNameReference(int ix)
+ throws IOException,JsonGenerationException
+ {
+ // 03-Mar-2011, tatu: Related to [JACKSON-525], let's add a sanity check here
+ if (ix >= _seenNameCount) {
+ throw new IllegalArgumentException("Internal error: trying to write shared name with index "+ix
+ +"; but have only seen "+_seenNameCount+" so far!");
+ }
+ if (ix < 64) {
+ _writeByte((byte) (TOKEN_PREFIX_KEY_SHARED_SHORT + ix));
+ } else {
+ _writeBytes(((byte) (TOKEN_PREFIX_KEY_SHARED_LONG + (ix >> 8))), (byte) ix);
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Output method implementations, textual
+ /**********************************************************
+ */
+
+ @Override
+ public void writeString(String text) throws IOException,JsonGenerationException
+ {
+ if (text == null) {
+ writeNull();
+ return;
+ }
+ _verifyValueWrite("write String value");
+ int len = text.length();
+ if (len == 0) {
+ _writeByte(TOKEN_LITERAL_EMPTY_STRING);
+ return;
+ }
+ // Longer string handling off-lined
+ if (len > MAX_SHARED_STRING_LENGTH_BYTES) {
+ _writeNonSharedString(text, len);
+ return;
+ }
+ // Then: is it something we can share?
+ if (_seenStringValueCount >= 0) {
+ int ix = _findSeenStringValue(text);
+ if (ix >= 0) {
+ _writeSharedStringValueReference(ix);
+ return;
+ }
+ }
+
+ // possibly short string (but not necessarily)
+ // first: ensure we have enough space
+ if ((_outputTail + MIN_BUFFER_FOR_POSSIBLE_SHORT_STRING) >= _outputEnd) {
+ _flushBuffer();
+ }
+ // then let's copy String chars to char buffer, faster than using getChar (measured, profiled)
+ text.getChars(0, len, _charBuffer, 0);
+ int origOffset = _outputTail;
+ ++_outputTail; // to leave room for type token
+ int byteLen = _shortUTF8Encode(_charBuffer, 0, len);
+ if (byteLen <= MAX_SHORT_VALUE_STRING_BYTES) { // yes, is short indeed
+ // plus keep reference, if it could be shared:
+ if (_seenStringValueCount >= 0) {
+ _addSeenStringValue(text);
+ }
+ if (byteLen == len) { // and all ASCII
+ _outputBuffer[origOffset] = (byte) ((TOKEN_PREFIX_TINY_ASCII - 1) + byteLen);
+ } else { // not just ASCII
+ // note: since length 1 can not be used here, value range is offset by 2, not 1
+ _outputBuffer[origOffset] = (byte) ((TOKEN_PREFIX_TINY_UNICODE - 2) + byteLen);
+ }
+ } else { // nope, longer String
+ _outputBuffer[origOffset] = (byteLen == len) ? TOKEN_BYTE_LONG_STRING_ASCII
+ : SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE;
+ // and we will need String end marker byte
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+ }
+
+ private final void _writeSharedStringValueReference(int ix)
+ throws IOException,JsonGenerationException
+ {
+ // 03-Mar-2011, tatu: Related to [JACKSON-525], let's add a sanity check here
+ if (ix >= _seenStringValueCount) {
+ throw new IllegalArgumentException("Internal error: trying to write shared String value with index "+ix
+ +"; but have only seen "+_seenStringValueCount+" so far!");
+ }
+ if (ix < 31) { // add 1, as byte 0 is omitted
+ _writeByte((byte) (TOKEN_PREFIX_SHARED_STRING_SHORT + 1 + ix));
+ } else {
+ _writeBytes(((byte) (TOKEN_PREFIX_SHARED_STRING_LONG + (ix >> 8))), (byte) ix);
+ }
+ }
+
+ /**
+ * Helper method called to handle cases where String value to write is known
+ * to be long enough not to be shareable.
+ */
+ private final void _writeNonSharedString(final String text, final int len)
+ throws IOException,JsonGenerationException
+ {
+ // First: can we at least make a copy to char[]?
+ if (len > _charBufferLength) { // nope; need to skip copy step (alas; this is slower)
+ _writeByte(SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE);
+ _slowUTF8Encode(text);
+ _writeByte(BYTE_MARKER_END_OF_STRING);
+ return;
+ }
+ text.getChars(0, len, _charBuffer, 0);
+ // Expansion can be 3x for Unicode; and then there's type byte and end marker, so:
+ int maxLen = len + len + len + 2;
+ // Next: does it always fit within output buffer?
+ if (maxLen > _outputBuffer.length) { // nope
+ // can't rewrite type buffer, so can't speculate it might be all-ASCII
+ _writeByte(SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE);
+ _mediumUTF8Encode(_charBuffer, 0, len);
+ _writeByte(BYTE_MARKER_END_OF_STRING);
+ return;
+ }
+
+ if ((_outputTail + maxLen) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int origOffset = _outputTail;
+ // can't say for sure if it's ASCII or Unicode, so:
+ _writeByte(TOKEN_BYTE_LONG_STRING_ASCII);
+ int byteLen = _shortUTF8Encode(_charBuffer, 0, len);
+ // If not ASCII, fix type:
+ if (byteLen > len) {
+ _outputBuffer[origOffset] = SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE;
+ }
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+
+ @Override
+ public void writeString(char[] text, int offset, int len) throws IOException, JsonGenerationException
+ {
+ // Shared strings are tricky; easiest to just construct String, call the other method
+ if (len <= MAX_SHARED_STRING_LENGTH_BYTES && _seenStringValueCount >= 0 && len > 0) {
+ writeString(new String(text, offset, len));
+ return;
+ }
+ _verifyValueWrite("write String value");
+ if (len == 0) {
+ _writeByte(TOKEN_LITERAL_EMPTY_STRING);
+ return;
+ }
+ if (len <= MAX_SHORT_VALUE_STRING_BYTES) { // possibly short strings (not necessarily)
+ // first: ensure we have enough space
+ if ((_outputTail + MIN_BUFFER_FOR_POSSIBLE_SHORT_STRING) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int origOffset = _outputTail;
+ ++_outputTail; // to leave room for type token
+ int byteLen = _shortUTF8Encode(text, offset, offset+len);
+ byte typeToken;
+ if (byteLen <= MAX_SHORT_VALUE_STRING_BYTES) { // yes, is short indeed
+ if (byteLen == len) { // and all ASCII
+ typeToken = (byte) ((TOKEN_PREFIX_TINY_ASCII - 1) + byteLen);
+ } else { // not just ASCII
+ typeToken = (byte) ((TOKEN_PREFIX_TINY_UNICODE - 2) + byteLen);
+ }
+ } else { // nope, longer non-ASCII Strings
+ typeToken = SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE;
+ // and we will need String end marker byte
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ }
+ // and then sneak in type token now that know the details
+ _outputBuffer[origOffset] = typeToken;
+ } else { // "long" String, never shared
+ // but might still fit within buffer?
+ int maxLen = len + len + len + 2;
+ if (maxLen <= _outputBuffer.length) { // yes indeed
+ if ((_outputTail + maxLen) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int origOffset = _outputTail;
+ _writeByte(SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE);
+ int byteLen = _shortUTF8Encode(text, offset, offset+len);
+ // if it's ASCII, let's revise our type determination (to help decoder optimize)
+ if (byteLen == len) {
+ _outputBuffer[origOffset] = TOKEN_BYTE_LONG_STRING_ASCII;
+ }
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ } else {
+ _writeByte(SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE);
+ _mediumUTF8Encode(text, offset, offset+len);
+ _writeByte(BYTE_MARKER_END_OF_STRING);
+ }
+ }
+ }
+
+ @Override
+ public final void writeString(SerializableString sstr)
+ throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("write String value");
+ // First: is it empty?
+ String str = sstr.getValue();
+ int len = str.length();
+ if (len == 0) {
+ _writeByte(TOKEN_LITERAL_EMPTY_STRING);
+ return;
+ }
+ // Second: something we can share?
+ if (len <= MAX_SHARED_STRING_LENGTH_BYTES && _seenStringValueCount >= 0) {
+ int ix = _findSeenStringValue(str);
+ if (ix >= 0) {
+ _writeSharedStringValueReference(ix);
+ return;
+ }
+ }
+ // If not, use pre-encoded version
+ byte[] raw = sstr.asUnquotedUTF8();
+ final int byteLen = raw.length;
+
+ if (byteLen <= MAX_SHORT_VALUE_STRING_BYTES) { // short string
+ // first: ensure we have enough space
+ if ((_outputTail + byteLen + 1) >= _outputEnd) {
+ _flushBuffer();
+ }
+ // ASCII or Unicode?
+ int typeToken = (byteLen == len)
+ ? ((TOKEN_PREFIX_TINY_ASCII - 1) + byteLen)
+ : ((TOKEN_PREFIX_TINY_UNICODE - 2) + byteLen)
+ ;
+ _outputBuffer[_outputTail++] = (byte) typeToken;
+ System.arraycopy(raw, 0, _outputBuffer, _outputTail, byteLen);
+ _outputTail += byteLen;
+ // plus keep reference, if it could be shared:
+ if (_seenStringValueCount >= 0) {
+ _addSeenStringValue(sstr.getValue());
+ }
+ } else { // "long" String, never shared
+ // but might still fit within buffer?
+ byte typeToken = (byteLen == len) ? TOKEN_BYTE_LONG_STRING_ASCII
+ : SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE;
+ _writeByte(typeToken);
+ _writeBytes(raw, 0, raw.length);
+ _writeByte(BYTE_MARKER_END_OF_STRING);
+ }
+ }
+
+ @Override
+ public void writeRawUTF8String(byte[] text, int offset, int len)
+ throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("write String value");
+ // first: is it empty String?
+ if (len == 0) {
+ _writeByte(TOKEN_LITERAL_EMPTY_STRING);
+ return;
+ }
+ // Sanity check: shared-strings incompatible with raw String writing
+ if (_seenStringValueCount >= 0) {
+ throw new UnsupportedOperationException("Can not use direct UTF-8 write methods when 'Feature.CHECK_SHARED_STRING_VALUES' enabled");
+ }
+ /* Other practical limitation is that we do not really know if it might be
+ * ASCII or not; and figuring it out is rather slow. So, best we can do is
+ * to declare we do not know it is ASCII (i.e. "is Unicode").
+ */
+ if (len <= MAX_SHARED_STRING_LENGTH_BYTES) { // up to 65 Unicode bytes
+ // first: ensure we have enough space
+ if ((_outputTail + len) >= _outputEnd) { // bytes, plus one for type indicator
+ _flushBuffer();
+ }
+ /* 11-Feb-2011, tatu: As per [JACKSON-492], mininum length for "Unicode"
+ * String is 2; 1 byte length must be ASCII.
+ */
+ if (len == 1) {
+ _outputBuffer[_outputTail++] = TOKEN_PREFIX_TINY_ASCII; // length of 1 cancels out (len-1)
+ _outputBuffer[_outputTail++] = text[offset];
+ } else {
+ _outputBuffer[_outputTail++] = (byte) ((TOKEN_PREFIX_TINY_UNICODE - 2) + len);
+ System.arraycopy(text, offset, _outputBuffer, _outputTail, len);
+ _outputTail += len;
+ }
+ } else { // "long" String
+ // but might still fit within buffer?
+ int maxLen = len + len + len + 2;
+ if (maxLen <= _outputBuffer.length) { // yes indeed
+ if ((_outputTail + maxLen) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE;
+ System.arraycopy(text, offset, _outputBuffer, _outputTail, len);
+ _outputTail += len;
+ _outputBuffer[_outputTail++] = BYTE_MARKER_END_OF_STRING;
+ } else {
+ _writeByte(SmileConstants.TOKEN_MISC_LONG_TEXT_UNICODE);
+ _writeBytes(text, offset, len);
+ _writeByte(BYTE_MARKER_END_OF_STRING);
+ }
+ }
+ }
+
+ @Override
+ public final void writeUTF8String(byte[] text, int offset, int len)
+ throws IOException, JsonGenerationException
+ {
+ // Since no escaping is needed, same as 'writeRawUTF8String'
+ writeRawUTF8String(text, offset, len);
+ }
+
+ /*
+ /**********************************************************
+ /* Output method implementations, unprocessed ("raw")
+ /**********************************************************
+ */
+
+ @Override
+ public void writeRaw(String text) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRaw(String text, int offset, int len) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRaw(char[] text, int offset, int len) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRaw(char c) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRawValue(String text) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRawValue(String text, int offset, int len) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ @Override
+ public void writeRawValue(char[] text, int offset, int len) throws IOException, JsonGenerationException {
+ throw _notSupported();
+ }
+
+ /*
+ /**********************************************************
+ /* Output method implementations, base64-encoded binary
+ /**********************************************************
+ */
+
+ @Override
+ public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int len) throws IOException, JsonGenerationException
+ {
+ if (data == null) {
+ writeNull();
+ return;
+ }
+ _verifyValueWrite("write Binary value");
+ if (isEnabled(Feature.ENCODE_BINARY_AS_7BIT)) {
+ _writeByte(TOKEN_MISC_BINARY_7BIT);
+ _write7BitBinaryWithLength(data, offset, len);
+ } else {
+ _writeByte(TOKEN_MISC_BINARY_RAW);
+ _writePositiveVInt(len);
+ // raw is dead simple of course:
+ _writeBytes(data, offset, len);
+ }
+ }
+
+ @Override
+ public int writeBinary(InputStream data, int dataLength)
+ throws IOException, JsonGenerationException
+ {
+ // Smile requires knowledge of length in advance, since binary is length-prefixed
+ if (dataLength < 0) {
+ throw new UnsupportedOperationException("Must pass actual length for Smile encoded data");
+ }
+ _verifyValueWrite("write Binary value");
+ int missing;
+ if (isEnabled(Feature.ENCODE_BINARY_AS_7BIT)) {
+ _writeByte(TOKEN_MISC_BINARY_7BIT);
+ byte[] encodingBuffer = _ioContext.allocBase64Buffer();
+ try {
+ missing = _write7BitBinaryWithLength(data, dataLength, encodingBuffer);
+ } finally {
+ _ioContext.releaseBase64Buffer(encodingBuffer);
+ }
+ } else {
+ _writeByte(TOKEN_MISC_BINARY_RAW );
+ _writePositiveVInt(dataLength);
+ // raw is dead simple of course:
+ missing = _writeBytes(data, dataLength);
+ }
+ if (missing > 0) {
+ _reportError("Too few bytes available: missing "+missing+" bytes (out of "+dataLength+")");
+ }
+ return dataLength;
+ }
+
+ @Override
+ public int writeBinary(Base64Variant b64variant, InputStream data, int dataLength)
+ throws IOException, JsonGenerationException
+ {
+ return writeBinary(data, dataLength);
+ }
+
+ /*
+ /**********************************************************
+ /* Output method implementations, primitive
+ /**********************************************************
+ */
+
+ @Override
+ public void writeBoolean(boolean state) throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("write boolean value");
+ if (state) {
+ _writeByte(TOKEN_LITERAL_TRUE);
+ } else {
+ _writeByte(TOKEN_LITERAL_FALSE);
+ }
+ }
+
+ @Override
+ public void writeNull() throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("write null value");
+ _writeByte(TOKEN_LITERAL_NULL);
+ }
+
+ @Override
+ public void writeNumber(int i) throws IOException, JsonGenerationException
+ {
+ _verifyValueWrite("write number");
+ // First things first: let's zigzag encode number
+ i = SmileUtil.zigzagEncode(i);
+ // tiny (single byte) or small (type + 6-bit value) number?
+ if (i <= 0x3F && i >= 0) {
+ if (i <= 0x1F) { // tiny
+ _writeByte((byte) (TOKEN_PREFIX_SMALL_INT + i));
+ return;
+ }
+ // nope, just small, 2 bytes (type, 1-byte zigzag value) for 6 bit value
+ _writeBytes(TOKEN_BYTE_INT_32, (byte) (0x80 + i));
+ return;
+ }
+ // Ok: let's find minimal representation then
+ byte b0 = (byte) (0x80 + (i & 0x3F));
+ i >>>= 6;
+ if (i <= 0x7F) { // 13 bits is enough (== 3 byte total encoding)
+ _writeBytes(TOKEN_BYTE_INT_32, (byte) i, b0);
+ return;
+ }
+ byte b1 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_32, (byte) i, b1, b0);
+ return;
+ }
+ byte b2 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_32, (byte) i, b2, b1, b0);
+ return;
+ }
+ // no, need all 5 bytes
+ byte b3 = (byte) (i & 0x7F);
+ _writeBytes(TOKEN_BYTE_INT_32, (byte) (i >> 7), b3, b2, b1, b0);
+ }
+
+ @Override
+ public void writeNumber(long l) throws IOException, JsonGenerationException
+ {
+ // First: maybe 32 bits is enough?
+ if (l <= MAX_INT_AS_LONG && l >= MIN_INT_AS_LONG) {
+ writeNumber((int) l);
+ return;
+ }
+ _verifyValueWrite("write number");
+ // Then let's zigzag encode it
+
+ l = SmileUtil.zigzagEncode(l);
+ // Ok, well, we do know that 5 lowest-significant bytes are needed
+ int i = (int) l;
+ // 4 can be extracted from lower int
+ byte b0 = (byte) (0x80 + (i & 0x3F)); // sign bit set in the last byte
+ byte b1 = (byte) ((i >> 6) & 0x7F);
+ byte b2 = (byte) ((i >> 13) & 0x7F);
+ byte b3 = (byte) ((i >> 20) & 0x7F);
+ // fifth one is split between ints:
+ l >>>= 27;
+ byte b4 = (byte) (((int) l) & 0x7F);
+
+ // which may be enough?
+ i = (int) (l >> 7);
+ if (i == 0) {
+ _writeBytes(TOKEN_BYTE_INT_64, b4, b3, b2, b1, b0);
+ return;
+ }
+
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_64, (byte) i);
+ _writeBytes(b4, b3, b2, b1, b0);
+ return;
+ }
+ byte b5 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_64, (byte) i);
+ _writeBytes(b5, b4, b3, b2, b1, b0);
+ return;
+ }
+ byte b6 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_64, (byte) i, b6);
+ _writeBytes(b5, b4, b3, b2, b1, b0);
+ return;
+ }
+ byte b7 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _writeBytes(TOKEN_BYTE_INT_64, (byte) i, b7, b6);
+ _writeBytes(b5, b4, b3, b2, b1, b0);
+ return;
+ }
+ byte b8 = (byte) (i & 0x7F);
+ i >>= 7;
+ // must be done, with 10 bytes! (9 * 7 + 6 == 69 bits; only need 63)
+ _writeBytes(TOKEN_BYTE_INT_64, (byte) i, b8, b7, b6);
+ _writeBytes(b5, b4, b3, b2, b1, b0);
+ }
+
+ @Override
+ public void writeNumber(BigInteger v) throws IOException, JsonGenerationException
+ {
+ if (v == null) {
+ writeNull();
+ return;
+ }
+ _verifyValueWrite("write number");
+ // quite simple: type, and then VInt-len prefixed 7-bit encoded binary data:
+ _writeByte(TOKEN_BYTE_BIG_INTEGER);
+ byte[] data = v.toByteArray();
+ _write7BitBinaryWithLength(data, 0, data.length);
+ }
+
+ @Override
+ public void writeNumber(double d) throws IOException, JsonGenerationException
+ {
+ // Ok, now, we needed token type byte plus 10 data bytes (7 bits each)
+ _ensureRoomForOutput(11);
+ _verifyValueWrite("write number");
+ /* 17-Apr-2010, tatu: could also use 'doubleToIntBits', but it seems more accurate to use
+ * exact representation; and possibly faster. However, if there are cases
+ * where collapsing of NaN was needed (for non-Java clients), this can
+ * be changed
+ */
+ long l = Double.doubleToRawLongBits(d);
+ _outputBuffer[_outputTail++] = TOKEN_BYTE_FLOAT_64;
+ // Handle first 29 bits (single bit first, then 4 x 7 bits)
+ int hi5 = (int) (l >>> 35);
+ _outputBuffer[_outputTail+4] = (byte) (hi5 & 0x7F);
+ hi5 >>= 7;
+ _outputBuffer[_outputTail+3] = (byte) (hi5 & 0x7F);
+ hi5 >>= 7;
+ _outputBuffer[_outputTail+2] = (byte) (hi5 & 0x7F);
+ hi5 >>= 7;
+ _outputBuffer[_outputTail+1] = (byte) (hi5 & 0x7F);
+ hi5 >>= 7;
+ _outputBuffer[_outputTail] = (byte) hi5;
+ _outputTail += 5;
+ // Then split byte (one that crosses lo/hi int boundary), 7 bits
+ {
+ int mid = (int) (l >> 28);
+ _outputBuffer[_outputTail++] = (byte) (mid & 0x7F);
+ }
+ // and then last 4 bytes (28 bits)
+ int lo4 = (int) l;
+ _outputBuffer[_outputTail+3] = (byte) (lo4 & 0x7F);
+ lo4 >>= 7;
+ _outputBuffer[_outputTail+2] = (byte) (lo4 & 0x7F);
+ lo4 >>= 7;
+ _outputBuffer[_outputTail+1] = (byte) (lo4 & 0x7F);
+ lo4 >>= 7;
+ _outputBuffer[_outputTail] = (byte) (lo4 & 0x7F);
+ _outputTail += 4;
+ }
+
+ @Override
+ public void writeNumber(float f) throws IOException, JsonGenerationException
+ {
+ // Ok, now, we needed token type byte plus 5 data bytes (7 bits each)
+ _ensureRoomForOutput(6);
+ _verifyValueWrite("write number");
+
+ /* 17-Apr-2010, tatu: could also use 'floatToIntBits', but it seems more accurate to use
+ * exact representation; and possibly faster. However, if there are cases
+ * where collapsing of NaN was needed (for non-Java clients), this can
+ * be changed
+ */
+ int i = Float.floatToRawIntBits(f);
+ _outputBuffer[_outputTail++] = TOKEN_BYTE_FLOAT_32;
+ _outputBuffer[_outputTail+4] = (byte) (i & 0x7F);
+ i >>= 7;
+ _outputBuffer[_outputTail+3] = (byte) (i & 0x7F);
+ i >>= 7;
+ _outputBuffer[_outputTail+2] = (byte) (i & 0x7F);
+ i >>= 7;
+ _outputBuffer[_outputTail+1] = (byte) (i & 0x7F);
+ i >>= 7;
+ _outputBuffer[_outputTail] = (byte) (i & 0x7F);
+ _outputTail += 5;
+ }
+
+ @Override
+ public void writeNumber(BigDecimal dec) throws IOException, JsonGenerationException
+ {
+ if (dec == null) {
+ writeNull();
+ return;
+ }
+ _verifyValueWrite("write number");
+ _writeByte(TOKEN_BYTE_BIG_DECIMAL);
+ int scale = dec.scale();
+ // Ok, first output scale as VInt
+ _writeSignedVInt(scale);
+ BigInteger unscaled = dec.unscaledValue();
+ byte[] data = unscaled.toByteArray();
+ // And then binary data in "safe" mode (7-bit values)
+ _write7BitBinaryWithLength(data, 0, data.length);
+ }
+
+ @Override
+ public void writeNumber(String encodedValue) throws IOException,JsonGenerationException, UnsupportedOperationException
+ {
+ /* 17-Apr-2010, tatu: Could try parsing etc; but for now let's not bother, it could
+ * just be some non-standard representation that caller wants to pass
+ */
+ throw _notSupported();
+ }
+
+ /*
+ /**********************************************************
+ /* Implementations for other methods
+ /**********************************************************
+ */
+
+ @Override
+ protected final void _verifyValueWrite(String typeMsg)
+ throws IOException, JsonGenerationException
+ {
+ int status = _writeContext.writeValue();
+ if (status == JsonWriteContext.STATUS_EXPECT_NAME) {
+ _reportError("Can not "+typeMsg+", expecting field name");
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Low-level output handling
+ /**********************************************************
+ */
+
+ @Override
+ public final void flush() throws IOException
+ {
+ _flushBuffer();
+ if (isEnabled(JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM)) {
+ _out.flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ /* 05-Dec-2008, tatu: To add [JACKSON-27], need to close open
+ * scopes.
+ */
+ // First: let's see that we still have buffers...
+ if (_outputBuffer != null
+ && isEnabled(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT)) {
+ while (true) {
+ JsonStreamContext ctxt = getOutputContext();
+ if (ctxt.inArray()) {
+ writeEndArray();
+ } else if (ctxt.inObject()) {
+ writeEndObject();
+ } else {
+ break;
+ }
+ }
+ }
+ boolean wasClosed = _closed;
+ super.close();
+
+ if (!wasClosed && isEnabled(Feature.WRITE_END_MARKER)) {
+ _writeByte(BYTE_MARKER_END_OF_CONTENT);
+ }
+ _flushBuffer();
+
+ if (_ioContext.isResourceManaged() || isEnabled(JsonGenerator.Feature.AUTO_CLOSE_TARGET)) {
+ _out.close();
+ } else {
+ // If we can't close it, we should at least flush
+ _out.flush();
+ }
+ // Internal buffer(s) generator has can now be released as well
+ _releaseBuffers();
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, UTF-8 encoding
+ /**********************************************************
+ */
+
+ /**
+ * Helper method called when the whole character sequence is known to
+ * fit in the output buffer regardless of UTF-8 expansion.
+ */
+ private final int _shortUTF8Encode(char[] str, int i, int end)
+ {
+ // First: let's see if it's all ASCII: that's rather fast
+ int ptr = _outputTail;
+ final byte[] outBuf = _outputBuffer;
+ do {
+ int c = str[i];
+ if (c > 0x7F) {
+ return _shortUTF8Encode2(str, i, end, ptr);
+ }
+ outBuf[ptr++] = (byte) c;
+ } while (++i < end);
+ int codedLen = ptr - _outputTail;
+ _outputTail = ptr;
+ return codedLen;
+ }
+
+ /**
+ * Helper method called when the whole character sequence is known to
+ * fit in the output buffer, but not all characters are single-byte (ASCII)
+ * characters.
+ */
+ private final int _shortUTF8Encode2(char[] str, int i, int end, int outputPtr)
+ {
+ final byte[] outBuf = _outputBuffer;
+ while (i < end) {
+ int c = str[i++];
+ if (c <= 0x7F) {
+ outBuf[outputPtr++] = (byte) c;
+ continue;
+ }
+ // Nope, multi-byte:
+ if (c < 0x800) { // 2-byte
+ outBuf[outputPtr++] = (byte) (0xc0 | (c >> 6));
+ outBuf[outputPtr++] = (byte) (0x80 | (c & 0x3f));
+ continue;
+ }
+ // 3 or 4 bytes (surrogate)
+ // Surrogates?
+ if (c < SURR1_FIRST || c > SURR2_LAST) { // nope, regular 3-byte character
+ outBuf[outputPtr++] = (byte) (0xe0 | (c >> 12));
+ outBuf[outputPtr++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ outBuf[outputPtr++] = (byte) (0x80 | (c & 0x3f));
+ continue;
+ }
+ // Yup, a surrogate pair
+ if (c > SURR1_LAST) { // must be from first range; second won't do
+ _throwIllegalSurrogate(c);
+ }
+ // ... meaning it must have a pair
+ if (i >= end) {
+ _throwIllegalSurrogate(c);
+ }
+ c = _convertSurrogate(c, str[i++]);
+ if (c > 0x10FFFF) { // illegal in JSON as well as in XML
+ _throwIllegalSurrogate(c);
+ }
+ outBuf[outputPtr++] = (byte) (0xf0 | (c >> 18));
+ outBuf[outputPtr++] = (byte) (0x80 | ((c >> 12) & 0x3f));
+ outBuf[outputPtr++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ outBuf[outputPtr++] = (byte) (0x80 | (c & 0x3f));
+ }
+ int codedLen = outputPtr - _outputTail;
+ _outputTail = outputPtr;
+ return codedLen;
+ }
+
+ private void _slowUTF8Encode(String str) throws IOException
+ {
+ final int len = str.length();
+ int inputPtr = 0;
+ final int bufferEnd = _outputEnd - 4;
+
+ output_loop:
+ for (; inputPtr < len; ) {
+ /* First, let's ensure we can output at least 4 bytes
+ * (longest UTF-8 encoded codepoint):
+ */
+ if (_outputTail >= bufferEnd) {
+ _flushBuffer();
+ }
+ int c = str.charAt(inputPtr++);
+ // And then see if we have an ASCII char:
+ if (c <= 0x7F) { // If so, can do a tight inner loop:
+ _outputBuffer[_outputTail++] = (byte)c;
+ // Let's calc how many ASCII chars we can copy at most:
+ int maxInCount = (len - inputPtr);
+ int maxOutCount = (bufferEnd - _outputTail);
+
+ if (maxInCount > maxOutCount) {
+ maxInCount = maxOutCount;
+ }
+ maxInCount += inputPtr;
+ ascii_loop:
+ while (true) {
+ if (inputPtr >= maxInCount) { // done with max. ascii seq
+ continue output_loop;
+ }
+ c = str.charAt(inputPtr++);
+ if (c > 0x7F) {
+ break ascii_loop;
+ }
+ _outputBuffer[_outputTail++] = (byte) c;
+ }
+ }
+
+ // Nope, multi-byte:
+ if (c < 0x800) { // 2-byte
+ _outputBuffer[_outputTail++] = (byte) (0xc0 | (c >> 6));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ } else { // 3 or 4 bytes
+ // Surrogates?
+ if (c < SURR1_FIRST || c > SURR2_LAST) {
+ _outputBuffer[_outputTail++] = (byte) (0xe0 | (c >> 12));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ continue;
+ }
+ // Yup, a surrogate:
+ if (c > SURR1_LAST) { // must be from first range
+ _throwIllegalSurrogate(c);
+ }
+ // and if so, followed by another from next range
+ if (inputPtr >= len) {
+ _throwIllegalSurrogate(c);
+ }
+ c = _convertSurrogate(c, str.charAt(inputPtr++));
+ if (c > 0x10FFFF) { // illegal, as per RFC 4627
+ _throwIllegalSurrogate(c);
+ }
+ _outputBuffer[_outputTail++] = (byte) (0xf0 | (c >> 18));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 12) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ }
+ }
+ }
+
+ private void _mediumUTF8Encode(char[] str, int inputPtr, int inputEnd) throws IOException
+ {
+ final int bufferEnd = _outputEnd - 4;
+
+ output_loop:
+ while (inputPtr < inputEnd) {
+ /* First, let's ensure we can output at least 4 bytes
+ * (longest UTF-8 encoded codepoint):
+ */
+ if (_outputTail >= bufferEnd) {
+ _flushBuffer();
+ }
+ int c = str[inputPtr++];
+ // And then see if we have an ASCII char:
+ if (c <= 0x7F) { // If so, can do a tight inner loop:
+ _outputBuffer[_outputTail++] = (byte)c;
+ // Let's calc how many ASCII chars we can copy at most:
+ int maxInCount = (inputEnd - inputPtr);
+ int maxOutCount = (bufferEnd - _outputTail);
+
+ if (maxInCount > maxOutCount) {
+ maxInCount = maxOutCount;
+ }
+ maxInCount += inputPtr;
+ ascii_loop:
+ while (true) {
+ if (inputPtr >= maxInCount) { // done with max. ascii seq
+ continue output_loop;
+ }
+ c = str[inputPtr++];
+ if (c > 0x7F) {
+ break ascii_loop;
+ }
+ _outputBuffer[_outputTail++] = (byte) c;
+ }
+ }
+
+ // Nope, multi-byte:
+ if (c < 0x800) { // 2-byte
+ _outputBuffer[_outputTail++] = (byte) (0xc0 | (c >> 6));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ } else { // 3 or 4 bytes
+ // Surrogates?
+ if (c < SURR1_FIRST || c > SURR2_LAST) {
+ _outputBuffer[_outputTail++] = (byte) (0xe0 | (c >> 12));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ continue;
+ }
+ // Yup, a surrogate:
+ if (c > SURR1_LAST) { // must be from first range
+ _throwIllegalSurrogate(c);
+ }
+ // and if so, followed by another from next range
+ if (inputPtr >= inputEnd) {
+ _throwIllegalSurrogate(c);
+ }
+ c = _convertSurrogate(c, str[inputPtr++]);
+ if (c > 0x10FFFF) { // illegal, as per RFC 4627
+ _throwIllegalSurrogate(c);
+ }
+ _outputBuffer[_outputTail++] = (byte) (0xf0 | (c >> 18));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 12) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ _outputBuffer[_outputTail++] = (byte) (0x80 | (c & 0x3f));
+ }
+ }
+ }
+
+ /**
+ * Method called to calculate UTF codepoint, from a surrogate pair.
+ */
+ private int _convertSurrogate(int firstPart, int secondPart)
+ {
+ // Ok, then, is the second part valid?
+ if (secondPart < SURR2_FIRST || secondPart > SURR2_LAST) {
+ throw new IllegalArgumentException("Broken surrogate pair: first char 0x"+Integer.toHexString(firstPart)+", second 0x"+Integer.toHexString(secondPart)+"; illegal combination");
+ }
+ return 0x10000 + ((firstPart - SURR1_FIRST) << 10) + (secondPart - SURR2_FIRST);
+ }
+
+ private void _throwIllegalSurrogate(int code)
+ {
+ if (code > 0x10FFFF) { // over max?
+ throw new IllegalArgumentException("Illegal character point (0x"+Integer.toHexString(code)+") to output; max is 0x10FFFF as per RFC 4627");
+ }
+ if (code >= SURR1_FIRST) {
+ if (code <= SURR1_LAST) { // Unmatched first part (closing without second part?)
+ throw new IllegalArgumentException("Unmatched first part of surrogate pair (0x"+Integer.toHexString(code)+")");
+ }
+ throw new IllegalArgumentException("Unmatched second part of surrogate pair (0x"+Integer.toHexString(code)+")");
+ }
+ // should we ever get this?
+ throw new IllegalArgumentException("Illegal character point (0x"+Integer.toHexString(code)+") to output");
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, writing bytes
+ /**********************************************************
+ */
+
+ private final void _ensureRoomForOutput(int needed) throws IOException
+ {
+ if ((_outputTail + needed) >= _outputEnd) {
+ _flushBuffer();
+ }
+ }
+
+ private final void _writeByte(byte b) throws IOException
+ {
+ if (_outputTail >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b;
+ }
+
+ private final void _writeBytes(byte b1, byte b2) throws IOException
+ {
+ if ((_outputTail + 1) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b2;
+ }
+
+ private final void _writeBytes(byte b1, byte b2, byte b3) throws IOException
+ {
+ if ((_outputTail + 2) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b3;
+ }
+
+ private final void _writeBytes(byte b1, byte b2, byte b3, byte b4) throws IOException
+ {
+ if ((_outputTail + 3) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b3;
+ _outputBuffer[_outputTail++] = b4;
+ }
+
+ private final void _writeBytes(byte b1, byte b2, byte b3, byte b4, byte b5) throws IOException
+ {
+ if ((_outputTail + 4) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b3;
+ _outputBuffer[_outputTail++] = b4;
+ _outputBuffer[_outputTail++] = b5;
+ }
+
+ private final void _writeBytes(byte b1, byte b2, byte b3, byte b4, byte b5, byte b6) throws IOException
+ {
+ if ((_outputTail + 5) >= _outputEnd) {
+ _flushBuffer();
+ }
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b3;
+ _outputBuffer[_outputTail++] = b4;
+ _outputBuffer[_outputTail++] = b5;
+ _outputBuffer[_outputTail++] = b6;
+ }
+
+ private final void _writeBytes(byte[] data, int offset, int len) throws IOException
+ {
+ if (len == 0) {
+ return;
+ }
+ if ((_outputTail + len) >= _outputEnd) {
+ _writeBytesLong(data, offset, len);
+ return;
+ }
+ // common case, non-empty, fits in just fine:
+ System.arraycopy(data, offset, _outputBuffer, _outputTail, len);
+ _outputTail += len;
+ }
+
+ private final int _writeBytes(InputStream in, int bytesLeft) throws IOException
+ {
+ while (bytesLeft > 0) {
+ int room = _outputEnd - _outputTail;
+ if (room < 0) {
+ _flushBuffer();
+ room = _outputEnd - _outputTail;
+ }
+ int count = in.read(_outputBuffer, _outputTail, room);
+ if (count < 0) {
+ break;
+ }
+ _outputTail += count;
+ bytesLeft -= count;
+ }
+ return bytesLeft;
+ }
+
+ private final void _writeBytesLong(byte[] data, int offset, int len) throws IOException
+ {
+ if (_outputTail >= _outputEnd) {
+ _flushBuffer();
+ }
+ while (true) {
+ int currLen = Math.min(len, (_outputEnd - _outputTail));
+ System.arraycopy(data, offset, _outputBuffer, _outputTail, currLen);
+ _outputTail += currLen;
+ if ((len -= currLen) == 0) {
+ break;
+ }
+ offset += currLen;
+ _flushBuffer();
+ }
+ }
+
+ /**
+ * Helper method for writing a 32-bit positive (really 31-bit then) value.
+ * Value is NOT zigzag encoded (since there is no sign bit to worry about)
+ */
+ private void _writePositiveVInt(int i) throws IOException
+ {
+ // At most 5 bytes (4 * 7 + 6 bits == 34 bits)
+ _ensureRoomForOutput(5);
+ byte b0 = (byte) (0x80 + (i & 0x3F));
+ i >>= 6;
+ if (i <= 0x7F) { // 6 or 13 bits is enough (== 2 or 3 byte total encoding)
+ if (i > 0) {
+ _outputBuffer[_outputTail++] = (byte) i;
+ }
+ _outputBuffer[_outputTail++] = b0;
+ return;
+ }
+ byte b1 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _outputBuffer[_outputTail++] = (byte) i;
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b0;
+ } else {
+ byte b2 = (byte) (i & 0x7F);
+ i >>= 7;
+ if (i <= 0x7F) {
+ _outputBuffer[_outputTail++] = (byte) i;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b0;
+ } else {
+ byte b3 = (byte) (i & 0x7F);
+ _outputBuffer[_outputTail++] = (byte) (i >> 7);
+ _outputBuffer[_outputTail++] = b3;
+ _outputBuffer[_outputTail++] = b2;
+ _outputBuffer[_outputTail++] = b1;
+ _outputBuffer[_outputTail++] = b0;
+ }
+ }
+ }
+
+ /**
+ * Helper method for writing 32-bit signed value, using
+ * "zig zag encoding" (see protocol buffers for explanation -- basically,
+ * sign bit is moved as LSB, rest of value shifted left by one)
+ * coupled with basic variable length encoding
+ */
+ private void _writeSignedVInt(int input) throws IOException
+ {
+ _writePositiveVInt(SmileUtil.zigzagEncode(input));
+ }
+
+ protected void _write7BitBinaryWithLength(byte[] data, int offset, int len) throws IOException
+ {
+ _writePositiveVInt(len);
+ // first, let's handle full 7-byte chunks
+ while (len >= 7) {
+ if ((_outputTail + 8) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int i = data[offset++]; // 1st byte
+ _outputBuffer[_outputTail++] = (byte) ((i >> 1) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 2nd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 2) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 3rd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 3) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 4th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 4) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 5th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 5) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 6th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 6) & 0x7F);
+ i = (i << 8) | (data[offset++] & 0xFF); // 7th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 7) & 0x7F);
+ _outputBuffer[_outputTail++] = (byte) (i & 0x7F);
+ len -= 7;
+ }
+ // and then partial piece, if any
+ if (len > 0) {
+ // up to 6 bytes to output, resulting in at most 7 bytes (which can encode 49 bits)
+ if ((_outputTail + 7) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int i = data[offset++];
+ _outputBuffer[_outputTail++] = (byte) ((i >> 1) & 0x7F);
+ if (len > 1) {
+ i = ((i & 0x01) << 8) | (data[offset++] & 0xFF); // 2nd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 2) & 0x7F);
+ if (len > 2) {
+ i = ((i & 0x03) << 8) | (data[offset++] & 0xFF); // 3rd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 3) & 0x7F);
+ if (len > 3) {
+ i = ((i & 0x07) << 8) | (data[offset++] & 0xFF); // 4th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 4) & 0x7F);
+ if (len > 4) {
+ i = ((i & 0x0F) << 8) | (data[offset++] & 0xFF); // 5th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 5) & 0x7F);
+ if (len > 5) {
+ i = ((i & 0x1F) << 8) | (data[offset++] & 0xFF); // 6th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 6) & 0x7F);
+ _outputBuffer[_outputTail++] = (byte) (i & 0x3F); // last 6 bits
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x1F); // last 5 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x0F); // last 4 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x07); // last 3 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x03); // last 2 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x01); // last bit
+ }
+ }
+ }
+
+ protected int _write7BitBinaryWithLength(InputStream in, int bytesLeft, byte[] buffer)
+ throws IOException
+ {
+ _writePositiveVInt(bytesLeft);
+ int inputPtr = 0;
+ int inputEnd = 0;
+ int lastFullOffset = -7;
+
+ // first, let's handle full 7-byte chunks
+ while (bytesLeft >= 7) {
+ if (inputPtr > lastFullOffset) {
+ inputEnd = _readMore(in, buffer, inputPtr, inputEnd, bytesLeft);
+ inputPtr = 0;
+ if (inputEnd < 7) { // required to try to read to have at least 7 bytes
+ bytesLeft -= inputEnd; // just to give accurate error messages wrt how much was gotten
+ break;
+ }
+ lastFullOffset = inputEnd-7;
+ }
+ if ((_outputTail + 8) >= _outputEnd) {
+ _flushBuffer();
+ }
+ int i = buffer[inputPtr++]; // 1st byte
+ _outputBuffer[_outputTail++] = (byte) ((i >> 1) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 2nd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 2) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 3rd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 3) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 4th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 4) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 5th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 5) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 6th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 6) & 0x7F);
+ i = (i << 8) | (buffer[inputPtr++] & 0xFF); // 7th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 7) & 0x7F);
+ _outputBuffer[_outputTail++] = (byte) (i & 0x7F);
+ bytesLeft -= 7;
+ }
+
+ // and then partial piece, if any
+ if (bytesLeft > 0) {
+ // up to 6 bytes to output, resulting in at most 7 bytes (which can encode 49 bits)
+ if ((_outputTail + 7) >= _outputEnd) {
+ _flushBuffer();
+ }
+ inputEnd = _readMore(in, buffer, inputPtr, inputEnd, bytesLeft);
+ inputPtr = 0;
+ if (inputEnd > 0) { // yes, but do we have room for output?
+ bytesLeft -= inputEnd;
+ int i = buffer[inputPtr++];
+ _outputBuffer[_outputTail++] = (byte) ((i >> 1) & 0x7F);
+ if (inputEnd > 1) {
+ i = ((i & 0x01) << 8) | (buffer[inputPtr++] & 0xFF); // 2nd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 2) & 0x7F);
+ if (inputEnd > 2) {
+ i = ((i & 0x03) << 8) | (buffer[inputPtr++] & 0xFF); // 3rd
+ _outputBuffer[_outputTail++] = (byte) ((i >> 3) & 0x7F);
+ if (inputEnd > 3) {
+ i = ((i & 0x07) << 8) | (buffer[inputPtr++] & 0xFF); // 4th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 4) & 0x7F);
+ if (inputEnd > 4) {
+ i = ((i & 0x0F) << 8) | (buffer[inputPtr++] & 0xFF); // 5th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 5) & 0x7F);
+ if (inputEnd > 5) {
+ i = ((i & 0x1F) << 8) | (buffer[inputPtr++] & 0xFF); // 6th
+ _outputBuffer[_outputTail++] = (byte) ((i >> 6) & 0x7F);
+ _outputBuffer[_outputTail++] = (byte) (i & 0x3F); // last 6 bits
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x1F); // last 5 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x0F); // last 4 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x07); // last 3 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x03); // last 2 bits
+ }
+ } else {
+ _outputBuffer[_outputTail++] = (byte) (i & 0x01); // last bit
+ }
+ }
+ }
+ return bytesLeft;
+ }
+
+ private int _readMore(InputStream in,
+ byte[] readBuffer, int inputPtr, int inputEnd,
+ int maxRead) throws IOException
+ {
+ // anything to shift to front?
+ int i = 0;
+ while (inputPtr < inputEnd) {
+ readBuffer[i++] = readBuffer[inputPtr++];
+ }
+ inputPtr = 0;
+ inputEnd = i;
+
+ maxRead = Math.min(maxRead, readBuffer.length);
+
+ do {
+ /* 26-Feb-2013, tatu: Similar to jackson-core issue #55, need to ensure
+ * we have something to read.
+ */
+ int length = maxRead - inputEnd;
+ if (length == 0) {
+ break;
+ }
+ int count = in.read(readBuffer, inputEnd, length);
+ if (count < 0) {
+ return inputEnd;
+ }
+ inputEnd += count;
+ } while (inputEnd < 7);
+ return inputEnd;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, buffer handling
+ /**********************************************************
+ */
+
+ @Override
+ protected void _releaseBuffers()
+ {
+ byte[] buf = _outputBuffer;
+ if (buf != null && _bufferRecyclable) {
+ _outputBuffer = null;
+ _ioContext.releaseWriteEncodingBuffer(buf);
+ }
+ char[] cbuf = _charBuffer;
+ if (cbuf != null) {
+ _charBuffer = null;
+ _ioContext.releaseConcatBuffer(cbuf);
+ }
+ /* Ok: since clearing up of larger arrays is much slower,
+ * let's only recycle default-sized buffers...
+ */
+ {
+ SharedStringNode[] nameBuf = _seenNames;
+ if (nameBuf != null && nameBuf.length == SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH) {
+ _seenNames = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer; and note
+ * that since it's a hash area, must clear all
+ */
+ if (_seenNameCount > 0) {
+ Arrays.fill(nameBuf, null);
+ }
+ _smileBufferRecycler.releaseSeenNamesBuffer(nameBuf);
+ }
+ }
+ {
+ SharedStringNode[] valueBuf = _seenStringValues;
+ if (valueBuf != null && valueBuf.length == SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH) {
+ _seenStringValues = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer; and note
+ * that since it's a hash area, must clear all
+ */
+ if (_seenStringValueCount > 0) {
+ Arrays.fill(valueBuf, null);
+ }
+ _smileBufferRecycler.releaseSeenStringValuesBuffer(valueBuf);
+ }
+ }
+ }
+
+ protected final void _flushBuffer() throws IOException
+ {
+ if (_outputTail > 0) {
+ _bytesWritten += _outputTail;
+ _out.write(_outputBuffer, 0, _outputTail);
+ _outputTail = 0;
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, handling shared string "maps"
+ /**********************************************************
+ */
+
+ private final int _findSeenName(String name)
+ {
+ int hash = name.hashCode();
+ SharedStringNode head = _seenNames[hash & (_seenNames.length-1)];
+ if (head == null) {
+ return -1;
+ }
+ SharedStringNode node = head;
+ // first, identity match; assuming most of the time we get intern()ed String
+ // And do unrolled initial check; 90+% likelihood head node has all info we need:
+ if (node.value == name) {
+ return node.index;
+ }
+ while ((node = node.next) != null) {
+ if (node.value == name) {
+ return node.index;
+ }
+ }
+ // If not, equality check; we already know head is not null
+ node = head;
+ do {
+ String value = node.value;
+ if (value.hashCode() == hash && value.equals(name)) {
+ return node.index;
+ }
+ node = node.next;
+ } while (node != null);
+ return -1;
+ }
+
+ private final void _addSeenName(String name)
+ {
+ // first: do we need to expand?
+ if (_seenNameCount == _seenNames.length) {
+ if (_seenNameCount == MAX_SHARED_NAMES) { // we are too full, restart from empty
+ Arrays.fill(_seenNames, null);
+ _seenNameCount = 0;
+ } else { // we always start with modest default size (like 64), so expand to full
+ SharedStringNode[] old = _seenNames;
+ _seenNames = new SharedStringNode[MAX_SHARED_NAMES];
+ final int mask = MAX_SHARED_NAMES-1;
+ for (SharedStringNode node : old) {
+ while (node != null) {
+ int ix = node.value.hashCode() & mask;
+ SharedStringNode next = node.next;
+ node.next = _seenNames[ix];
+ _seenNames[ix] = node;
+ node = next;
+ }
+ }
+ }
+ }
+ // other than that, just slap it there
+ int ix = name.hashCode() & (_seenNames.length-1);
+ _seenNames[ix] = new SharedStringNode(name, _seenNameCount, _seenNames[ix]);
+ ++_seenNameCount;
+ }
+
+ private final int _findSeenStringValue(String text)
+ {
+ int hash = text.hashCode();
+ SharedStringNode head = _seenStringValues[hash & (_seenStringValues.length-1)];
+ if (head != null) {
+ SharedStringNode node = head;
+ // first, identity match; assuming most of the time we get intern()ed String
+ do {
+ if (node.value == text) {
+ return node.index;
+ }
+ node = node.next;
+ } while (node != null);
+ // and then comparison, if no match yet
+ node = head;
+ do {
+ String value = node.value;
+ if (value.hashCode() == hash && value.equals(text)) {
+ return node.index;
+ }
+ node = node.next;
+ } while (node != null);
+ }
+ return -1;
+ }
+
+ private final void _addSeenStringValue(String text)
+ {
+ // first: do we need to expand?
+ if (_seenStringValueCount == _seenStringValues.length) {
+ if (_seenStringValueCount == MAX_SHARED_STRING_VALUES) { // we are too full, restart from empty
+ Arrays.fill(_seenStringValues, null);
+ _seenStringValueCount = 0;
+ } else { // we always start with modest default size (like 64), so expand to full
+ SharedStringNode[] old = _seenStringValues;
+ _seenStringValues = new SharedStringNode[MAX_SHARED_STRING_VALUES];
+ final int mask = MAX_SHARED_STRING_VALUES-1;
+ for (SharedStringNode node : old) {
+ while (node != null) {
+ int ix = node.value.hashCode() & mask;
+ SharedStringNode next = node.next;
+ node.next = _seenStringValues[ix];
+ _seenStringValues[ix] = node;
+ node = next;
+ }
+ }
+ }
+ }
+ // other than that, just slap it there
+ int ix = text.hashCode() & (_seenStringValues.length-1);
+ _seenStringValues[ix] = new SharedStringNode(text, _seenStringValueCount, _seenStringValues[ix]);
+ ++_seenStringValueCount;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, error reporting
+ /**********************************************************
+ */
+
+ /**
+ * Method for accessing offset of the next byte within the whole output
+ * stream that this generator has produced.
+ */
+ protected long outputOffset() {
+ return _bytesWritten + _outputTail;
+ }
+
+ protected UnsupportedOperationException _notSupported() {
+ return new UnsupportedOperationException();
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParser.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParser.java
new file mode 100644
index 0000000..82e6c2b
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParser.java
@@ -0,0 +1,2673 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.ref.SoftReference;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.base.ParserBase;
+import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer;
+import com.fasterxml.jackson.core.sym.Name;
+
+import static com.fasterxml.jackson.dataformat.smile.SmileConstants.BYTE_MARKER_END_OF_STRING;
+
+public class SmileParser
+ extends ParserBase
+{
+ /**
+ * Enumeration that defines all togglable features for Smile generators.
+ */
+ public enum Feature {
+ /**
+ * Feature that determines whether 4-byte Smile header is mandatory in input,
+ * or optional. If enabled, it means that only input that starts with the header
+ * is accepted as valid; if disabled, header is optional. In latter case,r
+ * settings for content are assumed to be defaults.
+ */
+ REQUIRE_HEADER(true)
+ ;
+
+ final boolean _defaultState;
+ final int _mask;
+
+ /**
+ * Method that calculates bit set (flags) of all features that
+ * are enabled by default.
+ */
+ public static int collectDefaults()
+ {
+ int flags = 0;
+ for (Feature f : values()) {
+ if (f.enabledByDefault()) {
+ flags |= f.getMask();
+ }
+ }
+ return flags;
+ }
+
+ private Feature(boolean defaultState) {
+ _defaultState = defaultState;
+ _mask = (1 << ordinal());
+ }
+
+ public boolean enabledByDefault() { return _defaultState; }
+ public int getMask() { return _mask; }
+ }
+
+ private final static int[] NO_INTS = new int[0];
+
+ private final static String[] NO_STRINGS = new String[0];
+
+ /*
+ /**********************************************************
+ /* Configuration
+ /**********************************************************
+ */
+
+ /**
+ * Codec used for data binding when (if) requested.
+ */
+ protected ObjectCodec _objectCodec;
+
+ /**
+ * Flag that indicates whether content can legally have raw (unquoted)
+ * binary data. Since this information is included both in header and
+ * in actual binary data blocks there is redundancy, and we want to
+ * ensure settings are compliant. Using application may also want to
+ * know this setting in case it does some direct (random) access.
+ */
+ protected boolean _mayContainRawBinary;
+
+ /**
+ * Helper object used for low-level recycling of Smile-generator
+ * specific buffers.
+ */
+ final protected SmileBufferRecycler<String> _smileBufferRecycler;
+
+ /*
+ /**********************************************************
+ /* Input source config, state (from ex StreamBasedParserBase)
+ /**********************************************************
+ */
+
+ /**
+ * Input stream that can be used for reading more content, if one
+ * in use. May be null, if input comes just as a full buffer,
+ * or if the stream has been closed.
+ */
+ protected InputStream _inputStream;
+
+ /**
+ * Current buffer from which data is read; generally data is read into
+ * buffer from input source, but in some cases pre-loaded buffer
+ * is handed to the parser.
+ */
+ protected byte[] _inputBuffer;
+
+ /**
+ * Flag that indicates whether the input buffer is recycable (and
+ * needs to be returned to recycler once we are done) or not.
+ *<p>
+ * If it is not, it also means that parser can NOT modify underlying
+ * buffer.
+ */
+ protected boolean _bufferRecyclable;
+
+ /*
+ /**********************************************************
+ /* Additional parsing state
+ /**********************************************************
+ */
+
+ /**
+ * Flag that indicates that the current token has not yet
+ * been fully processed, and needs to be finished for
+ * some access (or skipped to obtain the next token)
+ */
+ protected boolean _tokenIncomplete = false;
+
+ /**
+ * Type byte of the current token
+ */
+ protected int _typeByte;
+
+ /**
+ * Specific flag that is set when we encountered a 32-bit
+ * floating point value; needed since numeric super classes do
+ * not track distinction between float and double, but Smile
+ * format does, and we want to retain that separation.
+ */
+ protected boolean _got32BitFloat;
+
+ /*
+ /**********************************************************
+ /* Symbol handling, decoding
+ /**********************************************************
+ */
+
+ /**
+ * Symbol table that contains field names encountered so far
+ */
+ final protected BytesToNameCanonicalizer _symbols;
+
+ /**
+ * Temporary buffer used for name parsing.
+ */
+ protected int[] _quadBuffer = NO_INTS;
+
+ /**
+ * Quads used for hash calculation
+ */
+ protected int _quad1, _quad2;
+
+ /**
+ * Array of recently seen field names, which may be back referenced
+ * by later fields.
+ * Defaults set to enable handling even if no header found.
+ */
+ protected String[] _seenNames = NO_STRINGS;
+
+ protected int _seenNameCount = 0;
+
+ /**
+ * Array of recently seen field names, which may be back referenced
+ * by later fields
+ * Defaults set to disable handling if no header found.
+ */
+ protected String[] _seenStringValues = null;
+
+ protected int _seenStringValueCount = -1;
+
+ /*
+ /**********************************************************
+ /* Thread-local recycling
+ /**********************************************************
+ */
+
+ /**
+ * <code>ThreadLocal</code> contains a {@link java.lang.ref.SoftReference}
+ * to a buffer recycler used to provide a low-cost
+ * buffer recycling for Smile-specific buffers.
+ */
+ final protected static ThreadLocal<SoftReference<SmileBufferRecycler<String>>> _smileRecyclerRef
+ = new ThreadLocal<SoftReference<SmileBufferRecycler<String>>>();
+
+ /*
+ /**********************************************************
+ /* Life-cycle
+ /**********************************************************
+ */
+
+ public SmileParser(IOContext ctxt, int parserFeatures, int smileFeatures,
+ ObjectCodec codec,
+ BytesToNameCanonicalizer sym,
+ InputStream in, byte[] inputBuffer, int start, int end,
+ boolean bufferRecyclable)
+ {
+ super(ctxt, parserFeatures);
+ _objectCodec = codec;
+ _symbols = sym;
+
+ _inputStream = in;
+ _inputBuffer = inputBuffer;
+ _inputPtr = start;
+ _inputEnd = end;
+ _bufferRecyclable = bufferRecyclable;
+
+ _tokenInputRow = -1;
+ _tokenInputCol = -1;
+ _smileBufferRecycler = _smileBufferRecycler();
+ }
+
+ @Override
+ public ObjectCodec getCodec() {
+ return _objectCodec;
+ }
+
+ @Override
+ public void setCodec(ObjectCodec c) {
+ _objectCodec = c;
+ }
+
+ /**
+ * Helper method called when it looks like input might contain the signature;
+ * and it is necessary to detect and handle signature to get configuration
+ * information it might have.
+ *
+ * @return True if valid signature was found and handled; false if not
+ */
+ protected boolean handleSignature(boolean consumeFirstByte, boolean throwException)
+ throws IOException, JsonParseException
+ {
+ if (consumeFirstByte) {
+ ++_inputPtr;
+ }
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ if (_inputBuffer[_inputPtr] != SmileConstants.HEADER_BYTE_2) {
+ if (throwException) {
+ _reportError("Malformed content: signature not valid, starts with 0x3a but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr])+", not 0x29");
+ }
+ return false;
+ }
+ if (++_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ if (_inputBuffer[_inputPtr] != SmileConstants.HEADER_BYTE_3) {
+ if (throwException) {
+ _reportError("Malformed content: signature not valid, starts with 0x3a, 0x29, but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr])+", not 0xA");
+ }
+ return false;
+ }
+ // Good enough; just need version info from 4th byte...
+ if (++_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ int versionBits = (ch >> 4) & 0x0F;
+ // but failure with version number is fatal, can not ignore
+ if (versionBits != SmileConstants.HEADER_VERSION_0) {
+ _reportError("Header version number bits (0x"+Integer.toHexString(versionBits)+") indicate unrecognized version; only 0x0 handled by parser");
+ }
+
+ // can avoid tracking names, if explicitly disabled
+ if ((ch & SmileConstants.HEADER_BIT_HAS_SHARED_NAMES) == 0) {
+ _seenNames = null;
+ _seenNameCount = -1;
+ }
+ // conversely, shared string values must be explicitly enabled
+ if ((ch & SmileConstants.HEADER_BIT_HAS_SHARED_STRING_VALUES) != 0) {
+ _seenStringValues = NO_STRINGS;
+ _seenStringValueCount = 0;
+ }
+ _mayContainRawBinary = ((ch & SmileConstants.HEADER_BIT_HAS_RAW_BINARY) != 0);
+ return true;
+ }
+
+ protected final static SmileBufferRecycler<String> _smileBufferRecycler()
+ {
+ SoftReference<SmileBufferRecycler<String>> ref = _smileRecyclerRef.get();
+ SmileBufferRecycler<String> br = (ref == null) ? null : ref.get();
+
+ if (br == null) {
+ br = new SmileBufferRecycler<String>();
+ _smileRecyclerRef.set(new SoftReference<SmileBufferRecycler<String>>(br));
+ }
+ return br;
+ }
+
+ /*
+ /**********************************************************
+ /* Versioned
+ /**********************************************************
+ */
+
+ @Override
+ public Version version() {
+ return PackageVersion.VERSION;
+ }
+
+ /*
+ /**********************************************************
+ /* Former StreamBasedParserBase methods
+ /**********************************************************
+ */
+
+ @Override
+ public int releaseBuffered(OutputStream out) throws IOException
+ {
+ int count = _inputEnd - _inputPtr;
+ if (count < 1) {
+ return 0;
+ }
+ // let's just advance ptr to end
+ int origPtr = _inputPtr;
+ out.write(_inputBuffer, origPtr, count);
+ return count;
+ }
+
+ @Override
+ public Object getInputSource() {
+ return _inputStream;
+ }
+
+ /**
+ * Overridden since we do not really have character-based locations,
+ * but we do have byte offset to specify.
+ */
+ @Override
+ public JsonLocation getTokenLocation()
+ {
+ // token location is correctly managed...
+ return new JsonLocation(_ioContext.getSourceReference(),
+ _tokenInputTotal, // bytes
+ -1, -1, (int) _tokenInputTotal); // char offset, line, column
+ }
+
+ /**
+ * Overridden since we do not really have character-based locations,
+ * but we do have byte offset to specify.
+ */
+ @Override
+ public JsonLocation getCurrentLocation()
+ {
+ final long offset = _currInputProcessed + _inputPtr;
+ return new JsonLocation(_ioContext.getSourceReference(),
+ offset, // bytes
+ -1, -1, (int) offset); // char offset, line, column
+ }
+
+ /*
+ /**********************************************************
+ /* Low-level reading, other
+ /**********************************************************
+ */
+
+ @Override
+ protected final boolean loadMore()
+ throws IOException
+ {
+ _currInputProcessed += _inputEnd;
+ //_currInputRowStart -= _inputEnd;
+
+ if (_inputStream != null) {
+ int count = _inputStream.read(_inputBuffer, 0, _inputBuffer.length);
+ if (count > 0) {
+ _inputPtr = 0;
+ _inputEnd = count;
+ return true;
+ }
+ // End of input
+ _closeInput();
+ // Should never return 0, so let's fail
+ if (count == 0) {
+ throw new IOException("InputStream.read() returned 0 characters when trying to read "+_inputBuffer.length+" bytes");
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Helper method that will try to load at least specified number bytes in
+ * input buffer, possible moving existing data around if necessary
+ *
+ * @since 1.6
+ */
+ protected final boolean _loadToHaveAtLeast(int minAvailable)
+ throws IOException
+ {
+ // No input stream, no leading (either we are closed, or have non-stream input source)
+ if (_inputStream == null) {
+ return false;
+ }
+ // Need to move remaining data in front?
+ int amount = _inputEnd - _inputPtr;
+ if (amount > 0 && _inputPtr > 0) {
+ _currInputProcessed += _inputPtr;
+ //_currInputRowStart -= _inputPtr;
+ System.arraycopy(_inputBuffer, _inputPtr, _inputBuffer, 0, amount);
+ _inputEnd = amount;
+ } else {
+ _inputEnd = 0;
+ }
+ _inputPtr = 0;
+ while (_inputEnd < minAvailable) {
+ int count = _inputStream.read(_inputBuffer, _inputEnd, _inputBuffer.length - _inputEnd);
+ if (count < 1) {
+ // End of input
+ _closeInput();
+ // Should never return 0, so let's fail
+ if (count == 0) {
+ throw new IOException("InputStream.read() returned 0 characters when trying to read "+amount+" bytes");
+ }
+ return false;
+ }
+ _inputEnd += count;
+ }
+ return true;
+ }
+
+ @Override
+ protected void _closeInput() throws IOException
+ {
+ /* 25-Nov-2008, tatus: As per [JACKSON-16] we are not to call close()
+ * on the underlying InputStream, unless we "own" it, or auto-closing
+ * feature is enabled.
+ */
+ if (_inputStream != null) {
+ if (_ioContext.isResourceManaged() || isEnabled(JsonParser.Feature.AUTO_CLOSE_SOURCE)) {
+ _inputStream.close();
+ }
+ _inputStream = null;
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Overridden methods
+ /**********************************************************
+ */
+
+ @Override
+ protected void _finishString() throws IOException, JsonParseException
+ {
+ // should never be called; but must be defined for superclass
+ _throwInternal();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ super.close();
+ // Merge found symbols, if any:
+ _symbols.release();
+ }
+
+ @Override
+ public boolean hasTextCharacters()
+ {
+ if (_currToken == JsonToken.VALUE_STRING) {
+ // yes; is or can be made available efficiently as char[]
+ return _textBuffer.hasTextAsCharacters();
+ }
+ if (_currToken == JsonToken.FIELD_NAME) {
+ // not necessarily; possible but:
+ return _nameCopied;
+ }
+ // other types, no benefit from accessing as char[]
+ return false;
+ }
+
+ /**
+ * Method called to release internal buffers owned by the base
+ * reader. This may be called along with {@link #_closeInput} (for
+ * example, when explicitly closing this reader instance), or
+ * separately (if need be).
+ */
+ @Override
+ protected void _releaseBuffers() throws IOException
+ {
+ super._releaseBuffers();
+ if (_bufferRecyclable) {
+ byte[] buf = _inputBuffer;
+ if (buf != null) {
+ _inputBuffer = null;
+ _ioContext.releaseReadIOBuffer(buf);
+ }
+ }
+ {
+ String[] nameBuf = _seenNames;
+ if (nameBuf != null && nameBuf.length > 0) {
+ _seenNames = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
+ * but we only need to clear up to count as it is not a hash area
+ */
+ if (_seenNameCount > 0) {
+ Arrays.fill(nameBuf, 0, _seenNameCount, null);
+ }
+ _smileBufferRecycler.releaseSeenNamesBuffer(nameBuf);
+ }
+ }
+ {
+ String[] valueBuf = _seenStringValues;
+ if (valueBuf != null && valueBuf.length > 0) {
+ _seenStringValues = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
+ * but we only need to clear up to count as it is not a hash area
+ */
+ if (_seenStringValueCount > 0) {
+ Arrays.fill(valueBuf, 0, _seenStringValueCount, null);
+ }
+ _smileBufferRecycler.releaseSeenStringValuesBuffer(valueBuf);
+ }
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Extended API
+ /**********************************************************
+ */
+
+ public boolean mayContainRawBinary() {
+ return _mayContainRawBinary;
+ }
+
+ /*
+ /**********************************************************
+ /* JsonParser impl
+ /**********************************************************
+ */
+
+ @Override
+ public JsonToken nextToken() throws IOException, JsonParseException
+ {
+ _numTypesValid = NR_UNKNOWN;
+ // For longer tokens (text, binary), we'll only read when requested
+ if (_tokenIncomplete) {
+ _skipIncomplete();
+ }
+ _tokenInputTotal = _currInputProcessed + _inputPtr;
+ // also: clear any data retained so far
+ _binaryValue = null;
+ // Two main modes: values, and field names.
+ if (_parsingContext.inObject() && _currToken != JsonToken.FIELD_NAME) {
+ return (_currToken = _handleFieldName());
+ }
+ if (_inputPtr >= _inputEnd) {
+ if (!loadMore()) {
+ _handleEOF();
+ /* NOTE: here we can and should close input, release buffers,
+ * since this is "hard" EOF, not a boundary imposed by
+ * header token.
+ */
+ close();
+ return (_currToken = null);
+ }
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ _typeByte = ch;
+ switch ((ch >> 5) & 0x7) {
+ case 0: // short shared string value reference
+ if (ch == 0) { // important: this is invalid, don't accept
+ _reportError("Invalid token byte 0x00");
+ }
+ return _handleSharedString(ch-1);
+
+ case 1: // simple literals, numbers
+ {
+ int typeBits = ch & 0x1F;
+ if (typeBits < 4) {
+ switch (typeBits) {
+ case 0x00:
+ _textBuffer.resetWithEmpty();
+ return (_currToken = JsonToken.VALUE_STRING);
+ case 0x01:
+ return (_currToken = JsonToken.VALUE_NULL);
+ case 0x02: // false
+ return (_currToken = JsonToken.VALUE_FALSE);
+ default: // 0x03 == true
+ return (_currToken = JsonToken.VALUE_TRUE);
+ }
+ }
+ // next 3 bytes define subtype
+ if (typeBits < 8) { // VInt (zigzag), BigInteger
+ if ((typeBits & 0x3) <= 0x2) { // 0x3 reserved (should never occur)
+ _tokenIncomplete = true;
+ _numTypesValid = 0;
+ return (_currToken = JsonToken.VALUE_NUMBER_INT);
+ }
+ break;
+ }
+ if (typeBits < 12) { // floating-point
+ int subtype = typeBits & 0x3;
+ if (subtype <= 0x2) { // 0x3 reserved (should never occur)
+ _tokenIncomplete = true;
+ _numTypesValid = 0;
+ _got32BitFloat = (subtype == 0);
+ return (_currToken = JsonToken.VALUE_NUMBER_FLOAT);
+ }
+ break;
+ }
+ if (typeBits == 0x1A) { // == 0x3A == ':' -> possibly header signature for next chunk?
+ if (handleSignature(false, false)) {
+ /* Ok, now; end-marker and header both imply doc boundary and a
+ * 'null token'; but if both are seen, they are collapsed.
+ * We can check this by looking at current token; if it's null,
+ * need to get non-null token
+ */
+ if (_currToken == null) {
+ return nextToken();
+ }
+ return (_currToken = null);
+ }
+ }
+ _reportError("Unrecognized token byte 0x3A (malformed segment header?");
+ }
+ // and everything else is reserved, for now
+ break;
+ case 2: // tiny ASCII
+ // fall through
+ case 3: // short ASCII
+ // fall through
+ case 4: // tiny Unicode
+ // fall through
+ case 5: // short Unicode
+ // No need to decode, unless we have to keep track of back-references (for shared string values)
+ _currToken = JsonToken.VALUE_STRING;
+ if (_seenStringValueCount >= 0) { // shared text values enabled
+ _addSeenStringValue();
+ } else {
+ _tokenIncomplete = true;
+ }
+ return _currToken;
+ case 6: // small integers; zigzag encoded
+ _numberInt = SmileUtil.zigzagDecode(ch & 0x1F);
+ _numTypesValid = NR_INT;
+ return (_currToken = JsonToken.VALUE_NUMBER_INT);
+ case 7: // binary/long-text/long-shared/start-end-markers
+ switch (ch & 0x1F) {
+ case 0x00: // long variable length ASCII
+ case 0x04: // long variable length unicode
+ _tokenIncomplete = true;
+ return (_currToken = JsonToken.VALUE_STRING);
+ case 0x08: // binary, 7-bit (0xE8)
+ _tokenIncomplete = true;
+ return (_currToken = JsonToken.VALUE_EMBEDDED_OBJECT);
+ case 0x0C: // long shared string (0xEC)
+ case 0x0D:
+ case 0x0E:
+ case 0x0F:
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ return _handleSharedString(((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF));
+ case 0x18: // START_ARRAY
+ _parsingContext = _parsingContext.createChildArrayContext(-1, -1);
+ return (_currToken = JsonToken.START_ARRAY);
+ case 0x19: // END_ARRAY
+ if (!_parsingContext.inArray()) {
+ _reportMismatchedEndMarker(']', '}');
+ }
+ _parsingContext = _parsingContext.getParent();
+ return (_currToken = JsonToken.END_ARRAY);
+ case 0x1A: // START_OBJECT
+ _parsingContext = _parsingContext.createChildObjectContext(-1, -1);
+ return (_currToken = JsonToken.START_OBJECT);
+ case 0x1B: // not used in this mode; would be END_OBJECT
+ _reportError("Invalid type marker byte 0xFB in value mode (would be END_OBJECT in key mode)");
+ case 0x1D: // binary, raw
+ _tokenIncomplete = true;
+ return (_currToken = JsonToken.VALUE_EMBEDDED_OBJECT);
+ case 0x1F: // 0xFF, end of content
+ return (_currToken = null);
+ }
+ break;
+ }
+ // If we get this far, type byte is corrupt
+ _reportError("Invalid type marker byte 0x"+Integer.toHexString(ch & 0xFF)+" for expected value token");
+ return null;
+ }
+
+ private final JsonToken _handleSharedString(int index)
+ throws IOException, JsonParseException
+ {
+ if (index >= _seenStringValueCount) {
+ _reportInvalidSharedStringValue(index);
+ }
+ _textBuffer.resetWithString(_seenStringValues[index]);
+ return (_currToken = JsonToken.VALUE_STRING);
+ }
+
+ private final void _addSeenStringValue()
+ throws IOException, JsonParseException
+ {
+ _finishToken();
+ if (_seenStringValueCount < _seenStringValues.length) {
+ // !!! TODO: actually only store char[], first time around?
+ _seenStringValues[_seenStringValueCount++] = _textBuffer.contentsAsString();
+ return;
+ }
+ _expandSeenStringValues();
+ }
+
+ private final void _expandSeenStringValues()
+ {
+ String[] oldShared = _seenStringValues;
+ int len = oldShared.length;
+ String[] newShared;
+ if (len == 0) {
+ newShared = _smileBufferRecycler.allocSeenStringValuesBuffer();
+ if (newShared == null) {
+ newShared = new String[SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH];
+ }
+ } else if (len == SmileConstants.MAX_SHARED_STRING_VALUES) { // too many? Just flush...
+ newShared = oldShared;
+ _seenStringValueCount = 0; // could also clear, but let's not yet bother
+ } else {
+ int newSize = (len == SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH) ? 256 : SmileConstants.MAX_SHARED_STRING_VALUES;
+ newShared = new String[newSize];
+ System.arraycopy(oldShared, 0, newShared, 0, oldShared.length);
+ }
+ _seenStringValues = newShared;
+ _seenStringValues[_seenStringValueCount++] = _textBuffer.contentsAsString();
+ }
+
+ // base impl is fine:
+ //public String getCurrentName() throws IOException, JsonParseException
+
+ @Override
+ public NumberType getNumberType()
+ throws IOException, JsonParseException
+ {
+ if (_got32BitFloat) {
+ return NumberType.FLOAT;
+ }
+ return super.getNumberType();
+ }
+
+ /*
+ /**********************************************************
+ /* Public API, traversal, nextXxxValue/nextFieldName
+ /**********************************************************
+ */
+
+ @Override
+ public boolean nextFieldName(SerializableString str)
+ throws IOException, JsonParseException
+ {
+ // Two parsing modes; can only succeed if expecting field name, so handle that first:
+ if (_parsingContext.inObject() && _currToken != JsonToken.FIELD_NAME) {
+ byte[] nameBytes = str.asQuotedUTF8();
+ final int byteLen = nameBytes.length;
+ // need room for type byte, name bytes, possibly end marker, so:
+ if ((_inputPtr + byteLen + 1) < _inputEnd) { // maybe...
+ int ptr = _inputPtr;
+ int ch = _inputBuffer[ptr++];
+ _typeByte = ch;
+ main_switch:
+ switch ((ch >> 6) & 3) {
+ case 0: // misc, including end marker
+ switch (ch) {
+ case 0x20: // empty String as name, legal if unusual
+ _currToken = JsonToken.FIELD_NAME;
+ _inputPtr = ptr;
+ _parsingContext.setCurrentName("");
+ return (byteLen == 0);
+ case 0x30: // long shared
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ {
+ int index = ((ch & 0x3) << 8) + (_inputBuffer[ptr++] & 0xFF);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ String name = _seenNames[index];
+ _parsingContext.setCurrentName(name);
+ _inputPtr = ptr;
+ _currToken = JsonToken.FIELD_NAME;
+ return (name.equals(str.getValue()));
+ }
+ //case 0x34: // long ASCII/Unicode name; let's not even try...
+ }
+ break;
+ case 1: // short shared, can fully process
+ {
+ int index = (ch & 0x3F);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ _parsingContext.setCurrentName(_seenNames[index]);
+ String name = _seenNames[index];
+ _parsingContext.setCurrentName(name);
+ _inputPtr = ptr;
+ _currToken = JsonToken.FIELD_NAME;
+ return (name.equals(str.getValue()));
+ }
+ case 2: // short ASCII
+ {
+ int len = 1 + (ch & 0x3f);
+ if (len == byteLen) {
+ int i = 0;
+ for (; i < len; ++i) {
+ if (nameBytes[i] != _inputBuffer[ptr+i]) {
+ break main_switch;
+ }
+ }
+ // yes, does match...
+ _inputPtr = ptr + len;
+ final String name = str.getValue();
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ _currToken = JsonToken.FIELD_NAME;
+ return true;
+ }
+ }
+ break;
+ case 3: // short Unicode
+ // all valid, except for 0xFF
+ {
+ int len = (ch & 0x3F);
+ if (len > 0x37) {
+ if (len == 0x3B) {
+ _currToken = JsonToken.END_OBJECT;
+ if (!_parsingContext.inObject()) {
+ _reportMismatchedEndMarker('}', ']');
+ }
+ _inputPtr = ptr;
+ _parsingContext = _parsingContext.getParent();
+ return false;
+ }
+ // error, but let's not worry about that here
+ break;
+ }
+ len += 2; // values from 2 to 57...
+ if (len == byteLen) {
+ int i = 0;
+ for (; i < len; ++i) {
+ if (nameBytes[i] != _inputBuffer[ptr+i]) {
+ break main_switch;
+ }
+ }
+ // yes, does match...
+ _inputPtr = ptr + len;
+ final String name = str.getValue();
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ _currToken = JsonToken.FIELD_NAME;
+ return true;
+ }
+ }
+ break;
+ }
+ }
+ // otherwise fall back to default processing:
+ JsonToken t = _handleFieldName();
+ _currToken = t;
+ return (t == JsonToken.FIELD_NAME) && str.getValue().equals(_parsingContext.getCurrentName());
+ }
+ // otherwise just fall back to default handling; should occur rarely
+ return (nextToken() == JsonToken.FIELD_NAME) && str.getValue().equals(getCurrentName());
+ }
+
+ @Override
+ public String nextTextValue()
+ throws IOException, JsonParseException
+ {
+ // can't get text value if expecting name, so
+ if (!_parsingContext.inObject() || _currToken == JsonToken.FIELD_NAME) {
+ if (_tokenIncomplete) {
+ _skipIncomplete();
+ }
+ int ptr = _inputPtr;
+ if (ptr >= _inputEnd) {
+ if (!loadMore()) {
+ _handleEOF();
+ close();
+ _currToken = null;
+ return null;
+ }
+ ptr = _inputPtr;
+ }
+ int ch = _inputBuffer[ptr++];
+ _tokenInputTotal = _currInputProcessed + _inputPtr;
+
+ // also: clear any data retained so far
+ _binaryValue = null;
+ _typeByte = ch;
+
+ switch ((ch >> 5) & 0x7) {
+ case 0: // short shared string value reference
+ if (ch == 0) { // important: this is invalid, don't accept
+ _reportError("Invalid token byte 0x00");
+ }
+ // _handleSharedString...
+ {
+ --ch;
+ if (ch >= _seenStringValueCount) {
+ _reportInvalidSharedStringValue(ch);
+ }
+ _inputPtr = ptr;
+ String text = _seenStringValues[ch];
+ _textBuffer.resetWithString(text);
+ _currToken = JsonToken.VALUE_STRING;
+ return text;
+ }
+
+ case 1: // simple literals, numbers
+ {
+ int typeBits = ch & 0x1F;
+ if (typeBits == 0x00) {
+ _inputPtr = ptr;
+ _textBuffer.resetWithEmpty();
+ _currToken = JsonToken.VALUE_STRING;
+ return "";
+ }
+ }
+ break;
+ case 2: // tiny ASCII
+ // fall through
+ case 3: // short ASCII
+ _currToken = JsonToken.VALUE_STRING;
+ _inputPtr = ptr;
+ _decodeShortAsciiValue(1 + (ch & 0x3F));
+ {
+ // No need to decode, unless we have to keep track of back-references (for shared string values)
+ String text;
+ if (_seenStringValueCount >= 0) { // shared text values enabled
+ if (_seenStringValueCount < _seenStringValues.length) {
+ text = _textBuffer.contentsAsString();
+ _seenStringValues[_seenStringValueCount++] = text;
+ } else {
+ _expandSeenStringValues();
+ text = _textBuffer.contentsAsString();
+ }
+ } else {
+ text = _textBuffer.contentsAsString();
+ }
+ return text;
+ }
+
+ case 4: // tiny Unicode
+ // fall through
+ case 5: // short Unicode
+ _currToken = JsonToken.VALUE_STRING;
+ _inputPtr = ptr;
+ _decodeShortUnicodeValue(2 + (ch & 0x3F));
+ {
+ // No need to decode, unless we have to keep track of back-references (for shared string values)
+ String text;
+ if (_seenStringValueCount >= 0) { // shared text values enabled
+ if (_seenStringValueCount < _seenStringValues.length) {
+ text = _textBuffer.contentsAsString();
+ _seenStringValues[_seenStringValueCount++] = text;
+ } else {
+ _expandSeenStringValues();
+ text = _textBuffer.contentsAsString();
+ }
+ } else {
+ text = _textBuffer.contentsAsString();
+ }
+ return text;
+ }
+ case 6: // small integers; zigzag encoded
+ break;
+ case 7: // binary/long-text/long-shared/start-end-markers
+ // TODO: support longer strings too?
+ /*
+ switch (ch & 0x1F) {
+ case 0x00: // long variable length ASCII
+ case 0x04: // long variable length unicode
+ _tokenIncomplete = true;
+ return (_currToken = JsonToken.VALUE_STRING);
+ case 0x08: // binary, 7-bit
+ break main;
+ case 0x0C: // long shared string
+ case 0x0D:
+ case 0x0E:
+ case 0x0F:
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ return _handleSharedString(((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF));
+ }
+ break;
+ */
+ break;
+ }
+ }
+ // otherwise fall back to generic handling:
+ return (nextToken() == JsonToken.VALUE_STRING) ? getText() : null;
+ }
+
+ @Override
+ public int nextIntValue(int defaultValue)
+ throws IOException, JsonParseException
+ {
+ if (nextToken() == JsonToken.VALUE_NUMBER_INT) {
+ return getIntValue();
+ }
+ return defaultValue;
+ }
+
+ @Override
+ public long nextLongValue(long defaultValue)
+ throws IOException, JsonParseException
+ {
+ if (nextToken() == JsonToken.VALUE_NUMBER_INT) {
+ return getLongValue();
+ }
+ return defaultValue;
+ }
+
+ @Override
+ public Boolean nextBooleanValue()
+ throws IOException, JsonParseException
+ {
+ switch (nextToken()) {
+ case VALUE_TRUE:
+ return Boolean.TRUE;
+ case VALUE_FALSE:
+ return Boolean.FALSE;
+ default:
+ return null;
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Public API, access to token information, text
+ /**********************************************************
+ */
+
+ /**
+ * Method for accessing textual representation of the current event;
+ * if no current event (before first call to {@link #nextToken}, or
+ * after encountering end-of-input), returns null.
+ * Method can be called for any event.
+ */
+ @Override
+ public String getText()
+ throws IOException, JsonParseException
+ {
+ if (_tokenIncomplete) {
+ _tokenIncomplete = false;
+ // Let's inline part of "_finishToken", common case
+ int tb = _typeByte;
+ int type = (tb >> 5) & 0x7;
+ if (type == 2 || type == 3) { // tiny & short ASCII
+ _decodeShortAsciiValue(1 + (tb & 0x3F));
+ return _textBuffer.contentsAsString();
+ }
+ if (type == 4 || type == 5) { // tiny & short Unicode
+ // short unicode; note, lengths 2 - 65 (off-by-one compared to ASCII)
+ _decodeShortUnicodeValue(2 + (tb & 0x3F));
+ return _textBuffer.contentsAsString();
+ }
+ _finishToken();
+ }
+ if (_currToken == JsonToken.VALUE_STRING) {
+ return _textBuffer.contentsAsString();
+ }
+ JsonToken t = _currToken;
+ if (t == null) { // null only before/after document
+ return null;
+ }
+ if (t == JsonToken.FIELD_NAME) {
+ return _parsingContext.getCurrentName();
+ }
+ if (t.isNumeric()) {
+ // TODO: optimize?
+ return getNumberValue().toString();
+ }
+ return _currToken.asString();
+ }
+
+ @Override
+ public char[] getTextCharacters()
+ throws IOException, JsonParseException
+ {
+ if (_currToken != null) { // null only before/after document
+ if (_tokenIncomplete) {
+ _finishToken();
+ }
+ switch (_currToken) {
+ case VALUE_STRING:
+ return _textBuffer.getTextBuffer();
+ case FIELD_NAME:
+ if (!_nameCopied) {
+ String name = _parsingContext.getCurrentName();
+ int nameLen = name.length();
+ if (_nameCopyBuffer == null) {
+ _nameCopyBuffer = _ioContext.allocNameCopyBuffer(nameLen);
+ } else if (_nameCopyBuffer.length < nameLen) {
+ _nameCopyBuffer = new char[nameLen];
+ }
+ name.getChars(0, nameLen, _nameCopyBuffer, 0);
+ _nameCopied = true;
+ }
+ return _nameCopyBuffer;
+
+ // fall through
+ case VALUE_NUMBER_INT:
+ case VALUE_NUMBER_FLOAT:
+ // TODO: optimize
+ return getNumberValue().toString().toCharArray();
+
+ default:
+ return _currToken.asCharArray();
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public int getTextLength()
+ throws IOException, JsonParseException
+ {
+ if (_currToken != null) { // null only before/after document
+ if (_tokenIncomplete) {
+ _finishToken();
+ }
+ switch (_currToken) {
+ case VALUE_STRING:
+ return _textBuffer.size();
+ case FIELD_NAME:
+ return _parsingContext.getCurrentName().length();
+ // fall through
+ case VALUE_NUMBER_INT:
+ case VALUE_NUMBER_FLOAT:
+ // TODO: optimize
+ return getNumberValue().toString().length();
+
+ default:
+ return _currToken.asCharArray().length;
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ public int getTextOffset() throws IOException, JsonParseException
+ {
+ return 0;
+ }
+
+ @Override
+ public String getValueAsString() throws IOException, JsonParseException
+ {
+ if (_currToken != JsonToken.VALUE_STRING) {
+ if (_currToken == null || _currToken == JsonToken.VALUE_NULL || !_currToken.isScalarValue()) {
+ return null;
+ }
+ }
+ return getText();
+ }
+
+ @Override
+ public String getValueAsString(String defaultValue) throws IOException, JsonParseException
+ {
+ if (_currToken != JsonToken.VALUE_STRING) {
+ if (_currToken == null || _currToken == JsonToken.VALUE_NULL || !_currToken.isScalarValue()) {
+ return defaultValue;
+ }
+ }
+ return getText();
+ }
+
+ /*
+ /**********************************************************
+ /* Public API, access to token information, binary
+ /**********************************************************
+ */
+
+ @Override
+ public byte[] getBinaryValue(Base64Variant b64variant)
+ throws IOException, JsonParseException
+ {
+ if (_tokenIncomplete) {
+ _finishToken();
+ }
+ if (_currToken != JsonToken.VALUE_EMBEDDED_OBJECT ) {
+ // Todo, maybe: support base64 for text?
+ _reportError("Current token ("+_currToken+") not VALUE_EMBEDDED_OBJECT, can not access as binary");
+ }
+ return _binaryValue;
+ }
+
+ @Override
+ public Object getEmbeddedObject()
+ throws IOException, JsonParseException
+ {
+ if (_tokenIncomplete) {
+ _finishToken();
+ }
+ if (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT ) {
+ return _binaryValue;
+ }
+ return null;
+ }
+
+ @Override
+ public int readBinaryValue(Base64Variant b64variant, OutputStream out)
+ throws IOException, JsonParseException
+ {
+ if (_currToken != JsonToken.VALUE_EMBEDDED_OBJECT ) {
+ // Todo, maybe: support base64 for text?
+ _reportError("Current token ("+_currToken+") not VALUE_EMBEDDED_OBJECT, can not access as binary");
+ }
+ // Ok, first, unlikely (but legal?) case where someone already requested binary data:
+ if (!_tokenIncomplete) {
+ if (_binaryValue == null) { // most likely already read...
+ return 0;
+ }
+ final int len = _binaryValue.length;
+ out.write(_binaryValue, 0, len);
+ return len;
+ }
+
+ // otherwise, handle, mark as complete
+ // first, raw inlined binary data (simple)
+ if (_typeByte == SmileConstants.TOKEN_MISC_BINARY_RAW) {
+ final int totalCount = _readUnsignedVInt();
+ int left = totalCount;
+ while (left > 0) {
+ int avail = _inputEnd - _inputPtr;
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ avail = _inputEnd - _inputPtr;
+ }
+ int count = Math.min(avail, left);
+ out.write(_inputBuffer, _inputPtr, count);
+ _inputPtr += count;
+ left -= count;
+ }
+ _tokenIncomplete = false;
+ return totalCount;
+ }
+ if (_typeByte != SmileConstants.TOKEN_MISC_BINARY_7BIT) {
+ _throwInternal();
+ }
+ // or, alternative, 7-bit encoded stuff:
+ final int totalCount = _readUnsignedVInt();
+ byte[] encodingBuffer = _ioContext.allocBase64Buffer();
+ try {
+ _readBinaryEncoded(out, totalCount, encodingBuffer);
+ } finally {
+ _ioContext.releaseBase64Buffer(encodingBuffer);
+ }
+ _tokenIncomplete = false;
+ return totalCount;
+ }
+
+ private void _readBinaryEncoded(OutputStream out, int length, byte[] buffer)
+ throws IOException, JsonParseException
+ {
+ int outPtr = 0;
+ final int lastSafeOut = buffer.length - 7;
+ // first handle all full 7/8 units
+ while (length > 7) {
+ if ((_inputEnd - _inputPtr) < 8) {
+ _loadToHaveAtLeast(8);
+ }
+ int i1 = (_inputBuffer[_inputPtr++] << 25)
+ + (_inputBuffer[_inputPtr++] << 18)
+ + (_inputBuffer[_inputPtr++] << 11)
+ + (_inputBuffer[_inputPtr++] << 4);
+ int x = _inputBuffer[_inputPtr++];
+ i1 += x >> 3;
+ int i2 = ((x & 0x7) << 21)
+ + (_inputBuffer[_inputPtr++] << 14)
+ + (_inputBuffer[_inputPtr++] << 7)
+ + _inputBuffer[_inputPtr++];
+ // Ok: got our 7 bytes, just need to split, copy
+ buffer[outPtr++] = (byte)(i1 >> 24);
+ buffer[outPtr++] = (byte)(i1 >> 16);
+ buffer[outPtr++] = (byte)(i1 >> 8);
+ buffer[outPtr++] = (byte)i1;
+ buffer[outPtr++] = (byte)(i2 >> 16);
+ buffer[outPtr++] = (byte)(i2 >> 8);
+ buffer[outPtr++] = (byte)i2;
+ length -= 7;
+ // ensure there's always room for at least 7 bytes more after looping:
+ if (outPtr > lastSafeOut) {
+ out.write(buffer, 0, outPtr);
+ outPtr = 0;
+ }
+ }
+ // and then leftovers: n+1 bytes to decode n bytes
+ if (length > 0) {
+ if ((_inputEnd - _inputPtr) < (length+1)) {
+ _loadToHaveAtLeast(length+1);
+ }
+ int value = _inputBuffer[_inputPtr++];
+ for (int i = 1; i < length; ++i) {
+ value = (value << 7) + _inputBuffer[_inputPtr++];
+ buffer[outPtr++] = (byte) (value >> (7 - i));
+ }
+ // last byte is different, has remaining 1 - 6 bits, right-aligned
+ value <<= length;
+ buffer[outPtr++] = (byte) (value + _inputBuffer[_inputPtr++]);
+ }
+ if (outPtr > 0) {
+ out.write(buffer, 0, outPtr);
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, field name parsing
+ /**********************************************************
+ */
+
+ /**
+ * Method that handles initial token type recognition for token
+ * that has to be either FIELD_NAME or END_OBJECT.
+ */
+ protected final JsonToken _handleFieldName() throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ // is this needed?
+ _typeByte = ch;
+ switch ((ch >> 6) & 3) {
+ case 0: // misc, including end marker
+ switch (ch) {
+ case 0x20: // empty String as name, legal if unusual
+ _parsingContext.setCurrentName("");
+ return JsonToken.FIELD_NAME;
+ case 0x30: // long shared
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int index = ((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ _parsingContext.setCurrentName(_seenNames[index]);
+ }
+ return JsonToken.FIELD_NAME;
+ case 0x34: // long ASCII/Unicode name
+ _handleLongFieldName();
+ return JsonToken.FIELD_NAME;
+ }
+ break;
+ case 1: // short shared, can fully process
+ {
+ int index = (ch & 0x3F);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ _parsingContext.setCurrentName(_seenNames[index]);
+ }
+ return JsonToken.FIELD_NAME;
+ case 2: // short ASCII
+ {
+ int len = 1 + (ch & 0x3f);
+ String name;
+ Name n = _findDecodedFromSymbols(len);
+ if (n != null) {
+ name = n.getName();
+ _inputPtr += len;
+ } else {
+ name = _decodeShortAsciiName(len);
+ name = _addDecodedToSymbols(len, name);
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ }
+ return JsonToken.FIELD_NAME;
+ case 3: // short Unicode
+ // all valid, except for 0xFF
+ ch &= 0x3F;
+ {
+ if (ch > 0x37) {
+ if (ch == 0x3B) {
+ if (!_parsingContext.inObject()) {
+ _reportMismatchedEndMarker('}', ']');
+ }
+ _parsingContext = _parsingContext.getParent();
+ return JsonToken.END_OBJECT;
+ }
+ } else {
+ final int len = ch + 2; // values from 2 to 57...
+ String name;
+ Name n = _findDecodedFromSymbols(len);
+ if (n != null) {
+ name = n.getName();
+ _inputPtr += len;
+ } else {
+ name = _decodeShortUnicodeName(len);
+ name = _addDecodedToSymbols(len, name);
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ return JsonToken.FIELD_NAME;
+ }
+ }
+ break;
+ }
+ // Other byte values are illegal
+ _reportError("Invalid type marker byte 0x"+Integer.toHexString(_typeByte)+" for expected field name (or END_OBJECT marker)");
+ return null;
+ }
+
+ /**
+ * Method called to try to expand shared name area to fit one more potentially
+ * shared String. If area is already at its biggest size, will just clear
+ * the area (by setting next-offset to 0)
+ */
+ private final String[] _expandSeenNames(String[] oldShared)
+ {
+ int len = oldShared.length;
+ String[] newShared;
+ if (len == 0) {
+ newShared = _smileBufferRecycler.allocSeenNamesBuffer();
+ if (newShared == null) {
+ newShared = new String[SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH];
+ }
+ } else if (len == SmileConstants.MAX_SHARED_NAMES) { // too many? Just flush...
+ newShared = oldShared;
+ _seenNameCount = 0; // could also clear, but let's not yet bother
+ } else {
+ int newSize = (len == SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH) ? 256 : SmileConstants.MAX_SHARED_NAMES;
+ newShared = new String[newSize];
+ System.arraycopy(oldShared, 0, newShared, 0, oldShared.length);
+ }
+ return newShared;
+ }
+
+ private final String _addDecodedToSymbols(int len, String name)
+ {
+ if (len < 5) {
+ return _symbols.addName(name, _quad1, 0).getName();
+ }
+ if (len < 9) {
+ return _symbols.addName(name, _quad1, _quad2).getName();
+ }
+ int qlen = (len + 3) >> 2;
+ return _symbols.addName(name, _quadBuffer, qlen).getName();
+ }
+
+ private final String _decodeShortAsciiName(int len)
+ throws IOException, JsonParseException
+ {
+ // note: caller ensures we have enough bytes available
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int outPtr = 0;
+ final byte[] inBuf = _inputBuffer;
+ int inPtr = _inputPtr;
+
+ // loop unrolling seems to help here:
+ for (int inEnd = inPtr + len - 3; inPtr < inEnd; ) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ int left = (len & 3);
+ if (left > 0) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 1) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 2) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ }
+ }
+ _inputPtr = inPtr;
+ _textBuffer.setCurrentLength(len);
+ return _textBuffer.contentsAsString();
+ }
+
+ /**
+ * Helper method used to decode short Unicode string, length for which actual
+ * length (in bytes) is known
+ *
+ * @param len Length between 1 and 64
+ */
+ private final String _decodeShortUnicodeName(int len)
+ throws IOException, JsonParseException
+ {
+ // note: caller ensures we have enough bytes available
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int inPtr = _inputPtr;
+ _inputPtr += len;
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ final byte[] inBuf = _inputBuffer;
+ for (int end = inPtr + len; inPtr < end; ) {
+ int i = inBuf[inPtr++] & 0xFF;
+ int code = codes[i];
+ if (code != 0) {
+ // trickiest one, need surrogate handling
+ switch (code) {
+ case 1:
+ i = ((i & 0x1F) << 6) | (inBuf[inPtr++] & 0x3F);
+ break;
+ case 2:
+ i = ((i & 0x0F) << 12)
+ | ((inBuf[inPtr++] & 0x3F) << 6)
+ | (inBuf[inPtr++] & 0x3F);
+ break;
+ case 3:
+ i = ((i & 0x07) << 18)
+ | ((inBuf[inPtr++] & 0x3F) << 12)
+ | ((inBuf[inPtr++] & 0x3F) << 6)
+ | (inBuf[inPtr++] & 0x3F);
+ // note: this is the codepoint value; need to split, too
+ i -= 0x10000;
+ outBuf[outPtr++] = (char) (0xD800 | (i >> 10));
+ i = 0xDC00 | (i & 0x3FF);
+ break;
+ default: // invalid
+ _reportError("Invalid byte "+Integer.toHexString(i)+" in short Unicode text block");
+ }
+ }
+ outBuf[outPtr++] = (char) i;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ return _textBuffer.contentsAsString();
+ }
+
+ // note: slightly edited copy of UTF8StreamParser.addName()
+ private final Name _decodeLongUnicodeName(int[] quads, int byteLen, int quadLen)
+ throws IOException, JsonParseException
+ {
+ int lastQuadBytes = byteLen & 3;
+ // Ok: must decode UTF-8 chars. No other validation SHOULD be needed (except bounds checks?)
+ /* Note: last quad is not correctly aligned (leading zero bytes instead
+ * need to shift a bit, instead of trailing). Only need to shift it
+ * for UTF-8 decoding; need revert for storage (since key will not
+ * be aligned, to optimize lookup speed)
+ */
+ int lastQuad;
+
+ if (lastQuadBytes < 4) {
+ lastQuad = quads[quadLen-1];
+ // 8/16/24 bit left shift
+ quads[quadLen-1] = (lastQuad << ((4 - lastQuadBytes) << 3));
+ } else {
+ lastQuad = 0;
+ }
+
+ char[] cbuf = _textBuffer.emptyAndGetCurrentSegment();
+ int cix = 0;
+
+ for (int ix = 0; ix < byteLen; ) {
+ int ch = quads[ix >> 2]; // current quad, need to shift+mask
+ int byteIx = (ix & 3);
+ ch = (ch >> ((3 - byteIx) << 3)) & 0xFF;
+ ++ix;
+
+ if (ch > 127) { // multi-byte
+ int needed;
+ if ((ch & 0xE0) == 0xC0) { // 2 bytes (0x0080 - 0x07FF)
+ ch &= 0x1F;
+ needed = 1;
+ } else if ((ch & 0xF0) == 0xE0) { // 3 bytes (0x0800 - 0xFFFF)
+ ch &= 0x0F;
+ needed = 2;
+ } else if ((ch & 0xF8) == 0xF0) { // 4 bytes; double-char with surrogates and all...
+ ch &= 0x07;
+ needed = 3;
+ } else { // 5- and 6-byte chars not valid chars
+ _reportInvalidInitial(ch);
+ needed = ch = 1; // never really gets this far
+ }
+ if ((ix + needed) > byteLen) {
+ _reportInvalidEOF(" in long field name");
+ }
+
+ // Ok, always need at least one more:
+ int ch2 = quads[ix >> 2]; // current quad, need to shift+mask
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ if (needed > 1) {
+ ch2 = quads[ix >> 2];
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ if (needed > 2) { // 4 bytes? (need surrogates on output)
+ ch2 = quads[ix >> 2];
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2 & 0xFF);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ }
+ }
+ if (needed > 2) { // surrogate pair? once again, let's output one here, one later on
+ ch -= 0x10000; // to normalize it starting with 0x0
+ if (cix >= cbuf.length) {
+ cbuf = _textBuffer.expandCurrentSegment();
+ }
+ cbuf[cix++] = (char) (0xD800 + (ch >> 10));
+ ch = 0xDC00 | (ch & 0x03FF);
+ }
+ }
+ if (cix >= cbuf.length) {
+ cbuf = _textBuffer.expandCurrentSegment();
+ }
+ cbuf[cix++] = (char) ch;
+ }
+
+ // Ok. Now we have the character array, and can construct the String
+ String baseName = new String(cbuf, 0, cix);
+ // And finally, un-align if necessary
+ if (lastQuadBytes < 4) {
+ quads[quadLen-1] = lastQuad;
+ }
+ return _symbols.addName(baseName, quads, quadLen);
+ }
+
+ private final void _handleLongFieldName() throws IOException, JsonParseException
+ {
+ // First: gather quads we need, looking for end marker
+ final byte[] inBuf = _inputBuffer;
+ int quads = 0;
+ int bytes = 0;
+ int q = 0;
+
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ byte b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 0;
+ break;
+ }
+ q = ((int) b) & 0xFF;
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 1;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 2;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 3;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (quads >= _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, _quadBuffer.length + 256); // grow by 1k
+ }
+ _quadBuffer[quads++] = q;
+ }
+ // and if we have more bytes, append those too
+ int byteLen = (quads << 2);
+ if (bytes > 0) {
+ if (quads >= _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, _quadBuffer.length + 256);
+ }
+ _quadBuffer[quads++] = q;
+ byteLen += bytes;
+ }
+
+ // Know this name already?
+ String name;
+ Name n = _symbols.findName(_quadBuffer, quads);
+ if (n != null) {
+ name = n.getName();
+ } else {
+ name = _decodeLongUnicodeName(_quadBuffer, byteLen, quads).getName();
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ }
+
+ /**
+ * Helper method for trying to find specified encoded UTF-8 byte sequence
+ * from symbol table; if successful avoids actual decoding to String
+ */
+ private final Name _findDecodedFromSymbols(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ // First: maybe we already have this name decoded?
+ if (len < 5) {
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ int q = inBuf[inPtr] & 0xFF;
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ }
+ _quad1 = q;
+ return _symbols.findName(q);
+ }
+ if (len < 9) {
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ // First quadbyte is easy
+ int q1 = (inBuf[inPtr] & 0xFF) << 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ q1 <<= 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ q1 <<= 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ int q2 = (inBuf[++inPtr] & 0xFF);
+ len -= 5;
+ if (len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ }
+ _quad1 = q1;
+ _quad2 = q2;
+ return _symbols.findName(q1, q2);
+ }
+ return _findDecodedMedium(len);
+ }
+
+ /**
+ * Method for locating names longer than 8 bytes (in UTF-8)
+ */
+ private final Name _findDecodedMedium(int len)
+ throws IOException, JsonParseException
+ {
+ // first, need enough buffer to store bytes as ints:
+ {
+ int bufLen = (len + 3) >> 2;
+ if (bufLen > _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, bufLen);
+ }
+ }
+ // then decode, full quads first
+ int offset = 0;
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ do {
+ int q = (inBuf[inPtr++] & 0xFF) << 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ q <<= 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ q <<= 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ _quadBuffer[offset++] = q;
+ } while ((len -= 4) > 3);
+ // and then leftovers
+ if (len > 0) {
+ int q = inBuf[inPtr] & 0xFF;
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ _quadBuffer[offset++] = q;
+ }
+ return _symbols.findName(_quadBuffer, offset);
+ }
+
+ private static int[] _growArrayTo(int[] arr, int minSize)
+ {
+ int[] newArray = new int[minSize + 4];
+ if (arr != null) {
+ // !!! TODO: JDK 1.6, Arrays.copyOf
+ System.arraycopy(arr, 0, newArray, 0, arr.length);
+ }
+ return newArray;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, secondary parsing
+ /**********************************************************
+ */
+
+ @Override
+ protected void _parseNumericValue(int expType)
+ throws IOException, JsonParseException
+ {
+ if (_tokenIncomplete) {
+ int tb = _typeByte;
+ // ensure we got a numeric type with value that is lazily parsed
+ if (((tb >> 5) & 0x7) != 1) {
+ _reportError("Current token ("+_currToken+") not numeric, can not use numeric value accessors");
+ }
+ _tokenIncomplete = false;
+ _finishNumberToken(tb);
+ }
+ }
+
+ /**
+ * Method called to finish parsing of a token so that token contents
+ * are retriable
+ */
+ protected void _finishToken()
+ throws IOException, JsonParseException
+ {
+ _tokenIncomplete = false;
+ int tb = _typeByte;
+
+ int type = ((tb >> 5) & 0x7);
+ if (type == 1) { // simple literals, numbers
+ _finishNumberToken(tb);
+ return;
+ }
+ if (type <= 3) { // tiny & short ASCII
+ _decodeShortAsciiValue(1 + (tb & 0x3F));
+ return;
+ }
+ if (type <= 5) { // tiny & short Unicode
+ // short unicode; note, lengths 2 - 65 (off-by-one compared to ASCII)
+ _decodeShortUnicodeValue(2 + (tb & 0x3F));
+ return;
+ }
+ if (type == 7) {
+ tb &= 0x1F;
+ // next 3 bytes define subtype
+ switch (tb >> 2) {
+ case 0: // long variable length ASCII
+ _decodeLongAscii();
+ return;
+ case 1: // long variable length Unicode
+ _decodeLongUnicode();
+ return;
+ case 2: // binary, 7-bit
+ _binaryValue = _read7BitBinaryWithLength();
+ return;
+ case 7: // binary, raw
+ _finishRawBinary();
+ return;
+ }
+ }
+ // sanity check
+ _throwInternal();
+ }
+
+ protected final void _finishNumberToken(int tb)
+ throws IOException, JsonParseException
+ {
+ tb &= 0x1F;
+ int type = (tb >> 2);
+ if (type == 1) { // VInt (zigzag) or BigDecimal
+ int subtype = tb & 0x03;
+ if (subtype == 0) { // (v)int
+ _finishInt();
+ } else if (subtype == 1) { // (v)long
+ _finishLong();
+ } else if (subtype == 2) {
+ _finishBigInteger();
+ } else {
+ _throwInternal();
+ }
+ return;
+ }
+ if (type == 2) { // other numbers
+ switch (tb & 0x03) {
+ case 0: // float
+ _finishFloat();
+ return;
+ case 1: // double
+ _finishDouble();
+ return;
+ case 2: // big-decimal
+ _finishBigDecimal();
+ return;
+ }
+ }
+ _throwInternal();
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, secondary Number parsing
+ /**********************************************************
+ */
+
+ private final void _finishInt() throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int value = _inputBuffer[_inputPtr++];
+ int i;
+ if (value < 0) { // 6 bits
+ value &= 0x3F;
+ } else {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = _inputBuffer[_inputPtr++];
+ if (i >= 0) { // 13 bits
+ value = (value << 7) + i;
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = _inputBuffer[_inputPtr++];
+ if (i >= 0) {
+ value = (value << 7) + i;
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = _inputBuffer[_inputPtr++];
+ if (i >= 0) {
+ value = (value << 7) + i;
+ // and then we must get negative
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = _inputBuffer[_inputPtr++];
+ if (i >= 0) {
+ _reportError("Corrupt input; 32-bit VInt extends beyond 5 data bytes");
+ }
+ }
+ }
+ }
+ value = (value << 6) + (i & 0x3F);
+ }
+ _numberInt = SmileUtil.zigzagDecode(value);
+ _numTypesValid = NR_INT;
+ }
+
+ private final void _finishLong()
+ throws IOException, JsonParseException
+ {
+ // Ok, first, will always get 4 full data bytes first; 1 was already passed
+ long l = (long) _fourBytesToInt();
+ // and loop for the rest
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int value = _inputBuffer[_inputPtr++];
+ if (value < 0) {
+ l = (l << 6) + (value & 0x3F);
+ _numberLong = SmileUtil.zigzagDecode(l);
+ _numTypesValid = NR_LONG;
+ return;
+ }
+ l = (l << 7) + value;
+ }
+ }
+
+ private final void _finishBigInteger()
+ throws IOException, JsonParseException
+ {
+ byte[] raw = _read7BitBinaryWithLength();
+ _numberBigInt = new BigInteger(raw);
+ _numTypesValid = NR_BIGINT;
+ }
+
+ private final void _finishFloat()
+ throws IOException, JsonParseException
+ {
+ // just need 5 bytes to get int32 first; all are unsigned
+ int i = _fourBytesToInt();
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = (i << 7) + _inputBuffer[_inputPtr++];
+ float f = Float.intBitsToFloat(i);
+ _numberDouble = (double) f;
+ _numTypesValid = NR_DOUBLE;
+ }
+
+ private final void _finishDouble()
+ throws IOException, JsonParseException
+ {
+ // ok; let's take two sets of 4 bytes (each is int)
+ long hi = _fourBytesToInt();
+ long value = (hi << 28) + (long) _fourBytesToInt();
+ // and then remaining 2 bytes
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ value = (value << 7) + _inputBuffer[_inputPtr++];
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ value = (value << 7) + _inputBuffer[_inputPtr++];
+ _numberDouble = Double.longBitsToDouble(value);
+ _numTypesValid = NR_DOUBLE;
+ }
+
+ private final int _fourBytesToInt()
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int i = _inputBuffer[_inputPtr++]; // first 7 bits
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = (i << 7) + _inputBuffer[_inputPtr++]; // 14 bits
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ i = (i << 7) + _inputBuffer[_inputPtr++]; // 21
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ return (i << 7) + _inputBuffer[_inputPtr++];
+ }
+
+ private final void _finishBigDecimal()
+ throws IOException, JsonParseException
+ {
+ int scale = SmileUtil.zigzagDecode(_readUnsignedVInt());
+ byte[] raw = _read7BitBinaryWithLength();
+ _numberBigDecimal = new BigDecimal(new BigInteger(raw), scale);
+ _numTypesValid = NR_BIGDECIMAL;
+ }
+
+ private final int _readUnsignedVInt()
+ throws IOException, JsonParseException
+ {
+ int value = 0;
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int i = _inputBuffer[_inputPtr++];
+ if (i < 0) { // last byte
+ value = (value << 6) + (i & 0x3F);
+ return value;
+ }
+ value = (value << 7) + i;
+ }
+ }
+
+ private final byte[] _read7BitBinaryWithLength()
+ throws IOException, JsonParseException
+ {
+ int byteLen = _readUnsignedVInt();
+ byte[] result = new byte[byteLen];
+ int ptr = 0;
+ int lastOkPtr = byteLen - 7;
+
+ // first, read all 7-by-8 byte chunks
+ while (ptr <= lastOkPtr) {
+ if ((_inputEnd - _inputPtr) < 8) {
+ _loadToHaveAtLeast(8);
+ }
+ int i1 = (_inputBuffer[_inputPtr++] << 25)
+ + (_inputBuffer[_inputPtr++] << 18)
+ + (_inputBuffer[_inputPtr++] << 11)
+ + (_inputBuffer[_inputPtr++] << 4);
+ int x = _inputBuffer[_inputPtr++];
+ i1 += x >> 3;
+ int i2 = ((x & 0x7) << 21)
+ + (_inputBuffer[_inputPtr++] << 14)
+ + (_inputBuffer[_inputPtr++] << 7)
+ + _inputBuffer[_inputPtr++];
+ // Ok: got our 7 bytes, just need to split, copy
+ result[ptr++] = (byte)(i1 >> 24);
+ result[ptr++] = (byte)(i1 >> 16);
+ result[ptr++] = (byte)(i1 >> 8);
+ result[ptr++] = (byte)i1;
+ result[ptr++] = (byte)(i2 >> 16);
+ result[ptr++] = (byte)(i2 >> 8);
+ result[ptr++] = (byte)i2;
+ }
+ // and then leftovers: n+1 bytes to decode n bytes
+ int toDecode = (result.length - ptr);
+ if (toDecode > 0) {
+ if ((_inputEnd - _inputPtr) < (toDecode+1)) {
+ _loadToHaveAtLeast(toDecode+1);
+ }
+ int value = _inputBuffer[_inputPtr++];
+ for (int i = 1; i < toDecode; ++i) {
+ value = (value << 7) + _inputBuffer[_inputPtr++];
+ result[ptr++] = (byte) (value >> (7 - i));
+ }
+ // last byte is different, has remaining 1 - 6 bits, right-aligned
+ value <<= toDecode;
+ result[ptr] = (byte) (value + _inputBuffer[_inputPtr++]);
+ }
+ return result;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, secondary String parsing
+ /**********************************************************
+ */
+
+ protected final void _decodeShortAsciiValue(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ // Note: we count on fact that buffer must have at least 'len' (<= 64) empty char slots
+ final char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int outPtr = 0;
+ final byte[] inBuf = _inputBuffer;
+ int inPtr = _inputPtr;
+
+ // loop unrolling SHOULD be faster (as with _decodeShortAsciiName), but somehow
+ // is NOT; as per testing, benchmarking... very weird.
+ /*
+ for (int inEnd = inPtr + len - 3; inPtr < inEnd; ) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ int left = (len & 3);
+ if (left > 0) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 1) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 2) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ }
+ }
+ */
+
+ // meaning: regular tight loop is no slower, typically faster here:
+ for (final int end = inPtr + len; inPtr < end; ++inPtr) {
+ outBuf[outPtr++] = (char) inBuf[inPtr];
+ }
+
+ _inputPtr = inPtr;
+ _textBuffer.setCurrentLength(len);
+ }
+
+ protected final void _decodeShortUnicodeValue(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int inPtr = _inputPtr;
+ _inputPtr += len;
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ final byte[] inputBuf = _inputBuffer;
+ for (int end = inPtr + len; inPtr < end; ) {
+ int i = inputBuf[inPtr++] & 0xFF;
+ int code = codes[i];
+ if (code != 0) {
+ // trickiest one, need surrogate handling
+ switch (code) {
+ case 1:
+ i = ((i & 0x1F) << 6) | (inputBuf[inPtr++] & 0x3F);
+ break;
+ case 2:
+ i = ((i & 0x0F) << 12)
+ | ((inputBuf[inPtr++] & 0x3F) << 6)
+ | (inputBuf[inPtr++] & 0x3F);
+ break;
+ case 3:
+ i = ((i & 0x07) << 18)
+ | ((inputBuf[inPtr++] & 0x3F) << 12)
+ | ((inputBuf[inPtr++] & 0x3F) << 6)
+ | (inputBuf[inPtr++] & 0x3F);
+ // note: this is the codepoint value; need to split, too
+ i -= 0x10000;
+ outBuf[outPtr++] = (char) (0xD800 | (i >> 10));
+ i = 0xDC00 | (i & 0x3FF);
+ break;
+ default: // invalid
+ _reportError("Invalid byte "+Integer.toHexString(i)+" in short Unicode text block");
+ }
+ }
+ outBuf[outPtr++] = (char) i;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+
+ private final void _decodeLongAscii()
+ throws IOException, JsonParseException
+ {
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ main_loop:
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int inPtr = _inputPtr;
+ int left = _inputEnd - inPtr;
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ left = Math.min(left, outBuf.length - outPtr);
+ do {
+ byte b = _inputBuffer[inPtr++];
+ if (b == SmileConstants.BYTE_MARKER_END_OF_STRING) {
+ _inputPtr = inPtr;
+ break main_loop;
+ }
+ outBuf[outPtr++] = (char) b;
+ } while (--left > 0);
+ _inputPtr = inPtr;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+
+ private final void _decodeLongUnicode()
+ throws IOException, JsonParseException
+ {
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ int c;
+ final byte[] inputBuffer = _inputBuffer;
+
+ main_loop:
+ while (true) {
+ // First the tight ASCII loop:
+ ascii_loop:
+ while (true) {
+ int ptr = _inputPtr;
+ if (ptr >= _inputEnd) {
+ loadMoreGuaranteed();
+ ptr = _inputPtr;
+ }
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ int max = _inputEnd;
+ {
+ int max2 = ptr + (outBuf.length - outPtr);
+ if (max2 < max) {
+ max = max2;
+ }
+ }
+ while (ptr < max) {
+ c = (int) inputBuffer[ptr++] & 0xFF;
+ if (codes[c] != 0) {
+ _inputPtr = ptr;
+ break ascii_loop;
+ }
+ outBuf[outPtr++] = (char) c;
+ }
+ _inputPtr = ptr;
+ }
+ // Ok: end marker, escape or multi-byte?
+ if (c == SmileConstants.INT_MARKER_END_OF_STRING) {
+ break main_loop;
+ }
+
+ switch (codes[c]) {
+ case 1: // 2-byte UTF
+ c = _decodeUtf8_2(c);
+ break;
+ case 2: // 3-byte UTF
+ if ((_inputEnd - _inputPtr) >= 2) {
+ c = _decodeUtf8_3fast(c);
+ } else {
+ c = _decodeUtf8_3(c);
+ }
+ break;
+ case 3: // 4-byte UTF
+ c = _decodeUtf8_4(c);
+ // Let's add first part right away:
+ outBuf[outPtr++] = (char) (0xD800 | (c >> 10));
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ c = 0xDC00 | (c & 0x3FF);
+ // And let the other char output down below
+ break;
+ default:
+ // Is this good enough error message?
+ _reportInvalidChar(c);
+ }
+ // Need more room?
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ // Ok, let's add char to output:
+ outBuf[outPtr++] = (char) c;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+
+ private final void _finishRawBinary()
+ throws IOException, JsonParseException
+ {
+ int byteLen = _readUnsignedVInt();
+ _binaryValue = new byte[byteLen];
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ptr = 0;
+ while (true) {
+ int toAdd = Math.min(byteLen, _inputEnd - _inputPtr);
+ System.arraycopy(_inputBuffer, _inputPtr, _binaryValue, ptr, toAdd);
+ _inputPtr += toAdd;
+ ptr += toAdd;
+ byteLen -= toAdd;
+ if (byteLen <= 0) {
+ return;
+ }
+ loadMoreGuaranteed();
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, skipping
+ /**********************************************************
+ */
+
+ /**
+ * Method called to skip remainders of an incomplete token, when
+ * contents themselves will not be needed any more
+ */
+ protected void _skipIncomplete() throws IOException, JsonParseException
+ {
+ _tokenIncomplete = false;
+ int tb = _typeByte;
+ switch ((tb >> 5) & 0x7) {
+ case 1: // simple literals, numbers
+ tb &= 0x1F;
+ // next 3 bytes define subtype
+ switch (tb >> 2) {
+ case 1: // VInt (zigzag)
+ // easy, just skip until we see sign bit... (should we try to limit damage?)
+ switch (tb & 0x3) {
+ case 1: // vlong
+ _skipBytes(4); // min 5 bytes
+ // fall through
+ case 0: // vint
+ while (true) {
+ final int end = _inputEnd;
+ final byte[] buf = _inputBuffer;
+ while (_inputPtr < end) {
+ if (buf[_inputPtr++] < 0) {
+ return;
+ }
+ }
+ loadMoreGuaranteed();
+ }
+ case 2: // big-int
+ // just has binary data
+ _skip7BitBinary();
+ return;
+ }
+ break;
+ case 2: // other numbers
+ switch (tb & 0x3) {
+ case 0: // float
+ _skipBytes(5);
+ return;
+ case 1: // double
+ _skipBytes(10);
+ return;
+ case 2: // big-decimal
+ // first, skip scale
+ _readUnsignedVInt();
+ // then length-prefixed binary serialization
+ _skip7BitBinary();
+ return;
+ }
+ break;
+ }
+ break;
+ case 2: // tiny ASCII
+ // fall through
+ case 3: // short ASCII
+ _skipBytes(1 + (tb & 0x3F));
+ return;
+ case 4: // tiny unicode
+ // fall through
+ case 5: // short unicode
+ _skipBytes(2 + (tb & 0x3F));
+ return;
+ case 7:
+ tb &= 0x1F;
+ // next 3 bytes define subtype
+ switch (tb >> 2) {
+ case 0: // long variable length ASCII
+ case 1: // long variable length unicode
+ /* Doesn't matter which one, just need to find the end marker
+ * (note: can potentially skip invalid UTF-8 too)
+ */
+ while (true) {
+ final int end = _inputEnd;
+ final byte[] buf = _inputBuffer;
+ while (_inputPtr < end) {
+ if (buf[_inputPtr++] == BYTE_MARKER_END_OF_STRING) {
+ return;
+ }
+ }
+ loadMoreGuaranteed();
+ }
+ // never gets here
+ case 2: // binary, 7-bit
+ _skip7BitBinary();
+ return;
+ case 7: // binary, raw
+ _skipBytes(_readUnsignedVInt());
+ return;
+ }
+ }
+ _throwInternal();
+ }
+
+ protected void _skipBytes(int len)
+ throws IOException, JsonParseException
+ {
+ while (true) {
+ int toAdd = Math.min(len, _inputEnd - _inputPtr);
+ _inputPtr += toAdd;
+ len -= toAdd;
+ if (len <= 0) {
+ return;
+ }
+ loadMoreGuaranteed();
+ }
+ }
+
+ /**
+ * Helper method for skipping length-prefixed binary data
+ * section
+ */
+ protected void _skip7BitBinary()
+ throws IOException, JsonParseException
+ {
+ int origBytes = _readUnsignedVInt();
+ // Ok; 8 encoded bytes for 7 payload bytes first
+ int chunks = origBytes / 7;
+ int encBytes = chunks * 8;
+ // and for last 0 - 6 bytes, last+1 (except none if no leftovers)
+ origBytes -= 7 * chunks;
+ if (origBytes > 0) {
+ encBytes += 1 + origBytes;
+ }
+ _skipBytes(encBytes);
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, UTF8 decoding
+ /**********************************************************
+ */
+
+ private final int _decodeUtf8_2(int c)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ return ((c & 0x1F) << 6) | (d & 0x3F);
+ }
+
+ private final int _decodeUtf8_3(int c1)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ c1 &= 0x0F;
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ int c = (c1 << 6) | (d & 0x3F);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ return c;
+ }
+
+ private final int _decodeUtf8_3fast(int c1)
+ throws IOException, JsonParseException
+ {
+ c1 &= 0x0F;
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ int c = (c1 << 6) | (d & 0x3F);
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ return c;
+ }
+
+ /**
+ * @return Character value <b>minus 0x10000</c>; this so that caller
+ * can readily expand it to actual surrogates
+ */
+ private final int _decodeUtf8_4(int c)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = ((c & 0x07) << 6) | (d & 0x3F);
+
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+
+ /* note: won't change it to negative here, since caller
+ * already knows it'll need a surrogate
+ */
+ return ((c << 6) | (d & 0x3F)) - 0x10000;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, error reporting
+ /**********************************************************
+ */
+
+ protected void _reportInvalidSharedName(int index) throws IOException
+ {
+ if (_seenNames == null) {
+ _reportError("Encountered shared name reference, even though document header explicitly declared no shared name references are included");
+ }
+ _reportError("Invalid shared name reference "+index+"; only got "+_seenNameCount+" names in buffer (invalid content)");
+ }
+
+ protected void _reportInvalidSharedStringValue(int index) throws IOException
+ {
+ if (_seenStringValues == null) {
+ _reportError("Encountered shared text value reference, even though document header did not declared shared text value references may be included");
+ }
+ _reportError("Invalid shared text value reference "+index+"; only got "+_seenStringValueCount+" names in buffer (invalid content)");
+ }
+
+ protected void _reportInvalidChar(int c) throws JsonParseException
+ {
+ // Either invalid WS or illegal UTF-8 start char
+ if (c < ' ') {
+ _throwInvalidSpace(c);
+ }
+ _reportInvalidInitial(c);
+ }
+
+ protected void _reportInvalidInitial(int mask)
+ throws JsonParseException
+ {
+ _reportError("Invalid UTF-8 start byte 0x"+Integer.toHexString(mask));
+ }
+
+ protected void _reportInvalidOther(int mask)
+ throws JsonParseException
+ {
+ _reportError("Invalid UTF-8 middle byte 0x"+Integer.toHexString(mask));
+ }
+
+ protected void _reportInvalidOther(int mask, int ptr)
+ throws JsonParseException
+ {
+ _inputPtr = ptr;
+ _reportInvalidOther(mask);
+ }
+}
+
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParserBootstrapper.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParserBootstrapper.java
new file mode 100644
index 0000000..5e8b97f
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileParserBootstrapper.java
@@ -0,0 +1,262 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.format.InputAccessor;
+import com.fasterxml.jackson.core.format.MatchStrength;
+import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer;
+
+import static com.fasterxml.jackson.dataformat.smile.SmileConstants.*;
+
+/**
+ * Simple bootstrapper version used with Smile format parser.
+ */
+public class SmileParserBootstrapper
+{
+ /*
+ /**********************************************************
+ /* Configuration
+ /**********************************************************
+ */
+
+ protected final IOContext _context;
+
+ protected final InputStream _in;
+
+ /*
+ /**********************************************************
+ /* Input buffering
+ /**********************************************************
+ */
+
+ protected final byte[] _inputBuffer;
+
+ protected int _inputPtr;
+
+ protected int _inputEnd;
+
+ /**
+ * Flag that indicates whether buffer above is to be recycled
+ * after being used or not.
+ */
+ protected final boolean _bufferRecyclable;
+
+ /*
+ /**********************************************************
+ /* Input location
+ /**********************************************************
+ */
+
+ /**
+ * Current number of input units (bytes or chars) that were processed in
+ * previous blocks,
+ * before contents of current input buffer.
+ *<p>
+ * Note: includes possible BOMs, if those were part of the input.
+ */
+ protected int _inputProcessed;
+
+ /*
+ /**********************************************************
+ /* Life-cycle
+ /**********************************************************
+ */
+
+ public SmileParserBootstrapper(IOContext ctxt, InputStream in)
+ {
+ _context = ctxt;
+ _in = in;
+ _inputBuffer = ctxt.allocReadIOBuffer();
+ _inputEnd = _inputPtr = 0;
+ _inputProcessed = 0;
+ _bufferRecyclable = true;
+ }
+
+ public SmileParserBootstrapper(IOContext ctxt, byte[] inputBuffer, int inputStart, int inputLen)
+ {
+ _context = ctxt;
+ _in = null;
+ _inputBuffer = inputBuffer;
+ _inputPtr = inputStart;
+ _inputEnd = (inputStart + inputLen);
+ // Need to offset this for correct location info
+ _inputProcessed = -inputStart;
+ _bufferRecyclable = false;
+ }
+
+ public SmileParser constructParser(int generalParserFeatures, int smileFeatures,
+ boolean internNames,
+ ObjectCodec codec, BytesToNameCanonicalizer rootByteSymbols)
+ throws IOException, JsonParseException
+ {
+ BytesToNameCanonicalizer can = rootByteSymbols.makeChild(true, internNames);
+ // We just need a single byte, really, to know if it starts with header
+ ensureLoaded(1);
+ SmileParser p = new SmileParser(_context, generalParserFeatures, smileFeatures,
+ codec, can,
+ _in, _inputBuffer, _inputPtr, _inputEnd, _bufferRecyclable);
+ boolean hadSig = false;
+ if (_inputPtr < _inputEnd) { // only false for empty doc
+ if (_inputBuffer[_inputPtr] == SmileConstants.HEADER_BYTE_1) {
+ // need to ensure it gets properly handled so caller won't see the signature
+ hadSig = p.handleSignature(true, true);
+ }
+ } else {
+ /* 11-Oct-2012, tatu: Actually, let's allow empty documents even if
+ * header signature would otherwise be needed. This is useful for
+ * JAX-RS provider, empty PUT/POST payloads.
+ */
+ return p;
+ }
+ if (!hadSig && (smileFeatures & SmileParser.Feature.REQUIRE_HEADER.getMask()) != 0) {
+ // Ok, first, let's see if it looks like plain JSON...
+ String msg;
+
+ byte firstByte = (_inputPtr < _inputEnd) ? _inputBuffer[_inputPtr] : 0;
+ if (firstByte == '{' || firstByte == '[') {
+ msg = "Input does not start with Smile format header (first byte = 0x"
+ +Integer.toHexString(firstByte & 0xFF)+") -- rather, it starts with '"+((char) firstByte)
+ +"' (plain JSON input?) -- can not parse";
+ } else {
+ msg = "Input does not start with Smile format header (first byte = 0x"
+ +Integer.toHexString(firstByte & 0xFF)+") and parser has REQUIRE_HEADER enabled: can not parse";
+ }
+ throw new JsonParseException(msg, JsonLocation.NA);
+ }
+ return p;
+ }
+
+ /*
+ /**********************************************************
+ /* Encoding detection for data format auto-detection
+ /**********************************************************
+ */
+
+ public static MatchStrength hasSmileFormat(InputAccessor acc) throws IOException
+ {
+ // Ok: ideally we start with the header -- if so, we are golden
+ if (!acc.hasMoreBytes()) {
+ return MatchStrength.INCONCLUSIVE;
+ }
+ // We always need at least two bytes to determine, so
+ byte b1 = acc.nextByte();
+ if (!acc.hasMoreBytes()) {
+ return MatchStrength.INCONCLUSIVE;
+ }
+ byte b2 = acc.nextByte();
+
+ // First: do we see 3 "magic bytes"? If so, we are golden
+ if (b1 == SmileConstants.HEADER_BYTE_1) { // yeah, looks like marker
+ if (b2 != SmileConstants.HEADER_BYTE_2) {
+ return MatchStrength.NO_MATCH;
+ }
+ if (!acc.hasMoreBytes()) {
+ return MatchStrength.INCONCLUSIVE;
+ }
+ return (acc.nextByte() == SmileConstants.HEADER_BYTE_3) ?
+ MatchStrength.FULL_MATCH : MatchStrength.NO_MATCH;
+ }
+ // Otherwise: ideally either Object or Array:
+ if (b1 == SmileConstants.TOKEN_LITERAL_START_OBJECT) {
+ /* Object is bit easier, because now we need to get new name; i.e. can
+ * rule out name back-refs
+ */
+ if (b2 == SmileConstants.TOKEN_KEY_LONG_STRING) {
+ return MatchStrength.SOLID_MATCH;
+ }
+ int ch = (int) b2 & 0xFF;
+ if (ch >= 0x80 && ch < 0xF8) {
+ return MatchStrength.SOLID_MATCH;
+ }
+ return MatchStrength.NO_MATCH;
+ }
+ // Array bit trickier
+ if (b1 == SmileConstants.TOKEN_LITERAL_START_ARRAY) {
+ if (!acc.hasMoreBytes()) {
+ return MatchStrength.INCONCLUSIVE;
+ }
+ /* For arrays, we will actually accept much wider range of values (including
+ * things that could otherwise collide)
+ */
+ if (likelySmileValue(b2) || possibleSmileValue(b2, true)) {
+ return MatchStrength.SOLID_MATCH;
+ }
+ return MatchStrength.NO_MATCH;
+ }
+ // Scalar values are pretty weak, albeit possible; require more certain match, consider it weak:
+ if (likelySmileValue(b1) || possibleSmileValue(b2, false)) {
+ return MatchStrength.SOLID_MATCH;
+ }
+ return MatchStrength.NO_MATCH;
+ }
+
+ private static boolean likelySmileValue(byte b)
+ {
+ if ( (b == TOKEN_MISC_LONG_TEXT_ASCII) // 0xE0
+ || (b == TOKEN_MISC_LONG_TEXT_UNICODE) // 0xE4
+ || (b == TOKEN_MISC_BINARY_7BIT) // 0xE8
+ || (b == TOKEN_LITERAL_START_ARRAY) // 0xF8
+ || (b == TOKEN_LITERAL_START_OBJECT) // 0xFA
+ ) {
+ return true;
+ }
+ int ch = b & 0xFF;
+ // ASCII ctrl char range is pretty good match too
+ if (ch >= 0x80 && ch <= 0x9F) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @param lenient Whether to consider more speculative matches or not
+ * (typically true when there is context like start-array)
+ */
+ private static boolean possibleSmileValue(byte b, boolean lenient)
+ {
+ int ch = (int) b & 0xFF;
+ // note: we know that likely matches have been handled already, so...
+ if (ch >= 0x80) {
+ return (ch <= 0xE0);
+ }
+ if (lenient) {
+ if (ch >= 0x40) { // tiny/short ASCII
+ return true;
+ }
+ if (ch >- 0x20) { // various constants
+ return (ch < 0x2C); // many reserved bytes that can't be seen
+ }
+ }
+ return false;
+ }
+
+ /*
+ /**********************************************************
+ /* Internal methods, raw input access
+ /**********************************************************
+ */
+
+ protected boolean ensureLoaded(int minimum)
+ throws IOException
+ {
+ if (_in == null) { // block source; nothing more to load
+ return false;
+ }
+
+ /* Let's assume here buffer has enough room -- this will always
+ * be true for the limited used this method gets
+ */
+ int gotten = (_inputEnd - _inputPtr);
+ while (gotten < minimum) {
+ int count = _in.read(_inputBuffer, _inputEnd, _inputBuffer.length - _inputEnd);
+ if (count < 1) {
+ return false;
+ }
+ _inputEnd += count;
+ gotten += count;
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileUtil.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileUtil.java
new file mode 100644
index 0000000..ecf3843
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/SmileUtil.java
@@ -0,0 +1,46 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+/**
+ * Class for miscellaneous helper methods.
+ */
+public class SmileUtil
+{
+ public static int zigzagEncode(int input) {
+ // Canonical version:
+ //return (input << 1) ^ (input >> 31);
+ // but this is even better
+ if (input < 0) {
+ return (input << 1) ^ -1;
+ }
+ return (input << 1);
+ }
+
+ public static int zigzagDecode(int encoded) {
+ // canonical:
+ //return (encoded >>> 1) ^ (-(encoded & 1));
+ if ((encoded & 1) == 0) { // positive
+ return (encoded >>> 1);
+ }
+ // negative
+ return (encoded >>> 1) ^ -1;
+ }
+
+ public static long zigzagEncode(long input) {
+ // Canonical version
+ //return (input << 1) ^ (input >> 63);
+ if (input < 0L) {
+ return (input << 1) ^ -1L;
+ }
+ return (input << 1);
+ }
+
+ public static long zigzagDecode(long encoded) {
+ // canonical:
+ //return (encoded >>> 1) ^ (-(encoded & 1));
+ if ((encoded & 1) == 0) { // positive
+ return (encoded >>> 1);
+ }
+ // negative
+ return (encoded >>> 1) ^ -1L;
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/Tool.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/Tool.java
new file mode 100644
index 0000000..2f5647a
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/Tool.java
@@ -0,0 +1,160 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+/**
+ * Simple command-line utility that can be used to encode JSON as Smile, or
+ * decode JSON from Smile: direction is indicated by single command-line
+ * option of either "-e" (encode) or "-d" (decode).
+ *
+ * @author tatu
+ */
+public class Tool
+{
+ public final static String SUFFIX = ".lzf";
+
+ public final JsonFactory jsonFactory;
+ public final SmileFactory smileFactory;
+
+ public Tool()
+ {
+ jsonFactory = new JsonFactory();
+ smileFactory = new SmileFactory();
+ // check all shared refs (-> small size); add header, not trailing marker; do not use raw binary
+ smileFactory.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, true);
+ smileFactory.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+ smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, true);
+ smileFactory.configure(SmileGenerator.Feature.WRITE_HEADER, true);
+ smileFactory.configure(SmileGenerator.Feature.WRITE_END_MARKER, false);
+ // also: do not require header
+ smileFactory.configure(SmileParser.Feature.REQUIRE_HEADER, false);
+ }
+
+ private void process(String[] args) throws IOException
+ {
+ String oper = null;
+ String filename = null;
+
+ if (args.length == 2) {
+ oper = args[0];
+ filename = args[1];
+ } else if (args.length == 1) {
+ oper = args[0];
+ } else {
+ showUsage();
+ }
+
+ boolean encode = "-e".equals(oper);
+ if (encode) {
+ encode(inputStream(filename));
+ } else if ("-d".equals(oper)) {
+ decode(inputStream(filename));
+ } else if ("-v".equals(oper)) {
+ // need to read twice (encode, verify/compare)
+ verify(inputStream(filename), inputStream(filename));
+ } else {
+ showUsage();
+ }
+ }
+
+ private InputStream inputStream(String filename) throws IOException
+ {
+ // if no argument given, read from stdin
+ if (filename == null) {
+ return System.in;
+ }
+ File src = new File(filename);
+ if (!src.exists()) {
+ System.err.println("File '"+filename+"' does not exist.");
+ System.exit(1);
+ }
+ return new FileInputStream(src);
+ }
+
+ private void decode(InputStream in) throws IOException
+ {
+ JsonParser jp = smileFactory.createParser(in);
+ JsonGenerator jg = jsonFactory.createGenerator(System.out, JsonEncoding.UTF8);
+
+ while (true) {
+ /* Just one trick: since Smile can have segments (multiple 'documents' in output
+ * stream), we should not stop at first end marker, only bail out if two are seen
+ */
+ if (jp.nextToken() == null) {
+ if (jp.nextToken() == null) {
+ break;
+ }
+ }
+ jg.copyCurrentEvent(jp);
+ }
+ jp.close();
+ jg.close();
+ }
+
+ private void encode(InputStream in) throws IOException
+ {
+ JsonParser jp = jsonFactory.createParser(in);
+ JsonGenerator jg = smileFactory.createGenerator(System.out, JsonEncoding.UTF8);
+ while ((jp.nextToken()) != null) {
+ jg.copyCurrentEvent(jp);
+ }
+ jp.close();
+ jg.close();
+ }
+
+ private void verify(InputStream in, InputStream in2) throws IOException
+ {
+ JsonParser jp = jsonFactory.createParser(in);
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(4000);
+ JsonGenerator jg = smileFactory.createGenerator(bytes, JsonEncoding.UTF8);
+
+ // First, read, encode in memory buffer
+ while ((jp.nextToken()) != null) {
+ jg.copyCurrentEvent(jp);
+ }
+ jp.close();
+ jg.close();
+
+ // and then re-read both, verify
+ jp = jsonFactory.createParser(in2);
+ byte[] smile = bytes.toByteArray();
+ JsonParser jp2 = smileFactory.createParser(smile);
+
+ JsonToken t;
+ int count = 0;
+ while ((t = jp.nextToken()) != null) {
+ JsonToken t2 = jp2.nextToken();
+ ++count;
+ if (t != t2) {
+ throw new IOException("Input and encoded differ, token #"+count+"; expected "+t+", got "+t2);
+ }
+ // also, need to have same texts...
+ String text1 = jp.getText();
+ String text2 = jp2.getText();
+ if (!text1.equals(text2)) {
+ throw new IOException("Input and encoded differ, token #"+count+"; expected text '"+text1+"', got '"+text2+"'");
+ }
+ }
+
+ System.out.println("OK: verified "+count+" tokens (from "+smile.length+" bytes of Smile encoded data), input and encoded contents are identical");
+ }
+
+ protected void showUsage()
+ {
+ System.err.println("Usage: java "+getClass().getName()+" -e/-d [file]");
+ System.err.println(" (if no file given, reads from stdin -- always writes to stdout)");
+ System.err.println(" -d: decode Smile encoded input as JSON");
+ System.err.println(" -e: encode JSON (text) input as Smile");
+ System.err.println(" -v: encode JSON (text) input as Smile; read back, verify, do not write out");
+ System.exit(1);
+ }
+
+ public static void main(String[] args) throws IOException {
+ new Tool().process(args);
+ }
+
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingInputFeeder.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingInputFeeder.java
new file mode 100644
index 0000000..b688638
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingInputFeeder.java
@@ -0,0 +1,48 @@
+package com.fasterxml.jackson.dataformat.smile.async;
+
+import java.io.IOException;
+
+/**
+ * Interface used by non-blocking {@link com.fasterxml.jackson.core.JsonParser}
+ * to get more input to parse.
+ * It is accessed by entity that feeds content to parse; at any given point
+ * only one chunk of content can be processed so caller has to take care to
+ * only feed more content when existing content has been parsed (which occurs
+ * when parser's <code>nextToken</code> is called). Once application using
+ * non-blocking parser has no more data to feed it should call
+ * {@link #endOfInput} to indicate end of logical input stream.
+ *
+ * @author Tatu Saloranta
+ */
+public interface NonBlockingInputFeeder
+{
+ /**
+ * Method called to check whether it is ok to feed more data: parser returns true
+ * if it has no more content to parse (and it is ok to feed more); otherwise false
+ * (and no data should yet be fed).
+ */
+ public boolean needMoreInput();
+
+ /**
+ * Method that can be called to feed more data, if (and only if)
+ * {@link #needMoreInput} returns true.
+ *
+ * @param data Byte array that contains data to feed: caller must ensure data remains
+ * stable until it is fully processed (which is true when {@link #needMoreInput}
+ * returns true)
+ * @param offset Offset within array where input data to process starts
+ * @param len Length of input data within array to process.
+ *
+ * @throws IOException if the state is such that this method should not be called
+ * (has not yet consumed existing input data, or has been marked as closed)
+ */
+ public void feedInput(byte[] data, int offset, int len) throws IOException;
+
+ /**
+ * Method that should be called after last chunk of data to parse has been fed
+ * (with {@link #feedInput}); can be called regardless of what {@link #needMoreInput}
+ * returns. After calling this method, no more data can be fed; and parser assumes
+ * no more data will be available.
+ */
+ public void endOfInput();
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParser.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParser.java
new file mode 100644
index 0000000..4e42aa5
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParser.java
@@ -0,0 +1,31 @@
+package com.fasterxml.jackson.dataformat.smile.async;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonToken;
+
+/**
+ * Mix-in interface used with {@link com.fasterxml.jackson.core.JsonParser},
+ * extending it with features needed to process data in non-blocking
+ * ("asynchronous")
+ */
+public interface NonBlockingParser
+ extends NonBlockingInputFeeder
+{
+ /**
+ * Method that can be called when current token is not yet
+ * available via {@link com.fasterxml.jackson.core.JsonParser#getCurrentToken},
+ * to try to figure out what kind of token will be eventually returned
+ * once the whole token is decoded, if known.
+ * Note that this may return {@link com.fasterxml.jackson.core.JsonToken#NOT_AVAILABLE}:
+ * this occurs either if current token is known (and thus no more
+ * parsing can be done yet), or if not enough content is available
+ * to even determine next token type (typically we only need a single
+ * byte, but in boundaries zero bytes is available).
+ *
+ * @return Token that will eventually be returned with
+ * a call to {@link com.fasterxml.jackson.core.JsonParser#nextToken}, if known
+ */
+ public JsonToken peekNextToken() throws IOException, JsonParseException;
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParserImpl.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParserImpl.java
new file mode 100644
index 0000000..f42ab16
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/NonBlockingParserImpl.java
@@ -0,0 +1,2227 @@
+package com.fasterxml.jackson.dataformat.smile.async;
+
+import java.io.*;
+import java.lang.ref.SoftReference;
+//import java.math.BigDecimal;
+//import java.math.BigInteger;
+import java.util.Arrays;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.base.ParserBase;
+import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer;
+import com.fasterxml.jackson.core.sym.Name;
+import com.fasterxml.jackson.dataformat.smile.*;
+
+import static com.fasterxml.jackson.dataformat.smile.SmileConstants.BYTE_MARKER_END_OF_STRING;
+
+public class NonBlockingParserImpl
+ extends ParserBase
+ implements NonBlockingParser, NonBlockingInputFeeder
+{
+ private final static byte[] NO_BYTES = new byte[0];
+ private final static int[] NO_INTS = new int[0];
+ private final static String[] NO_STRINGS = new String[0];
+
+ /*
+ /**********************************************************************
+ /* State constants
+ /**********************************************************************
+ */
+
+ // // // Initial bootstrapping states:
+
+ /**
+ * State right after parser has been constructed: waiting for header
+ * (which may or may not be mandatory).
+ */
+ protected final static int STATE_INITIAL = 0;
+
+
+ /**
+ * State for recognized header marker, either in-feed or initial.
+ */
+ protected final static int STATE_HEADER = 1;
+
+ /**
+ * State in which we are right after decoding a full token.
+ */
+ protected final static int STATE_TOKEN_COMPLETE = 2;
+
+ // // // States for decoding numbers:
+ protected final static int STATE_NUMBER_INT = 10;
+ protected final static int STATE_NUMBER_LONG = 11;
+ protected final static int STATE_NUMBER_BIGINT = 12;
+ protected final static int STATE_NUMBER_FLOAT = 13;
+ protected final static int STATE_NUMBER_DOUBLE = 14;
+ protected final static int STATE_NUMBER_BIGDEC = 15;
+
+ protected final static int STATE_SHORT_ASCII = 20;
+ protected final static int STATE_SHORT_UNICODE = 21;
+ protected final static int STATE_LONG_ASCII = 22;
+ protected final static int STATE_LONG_UNICODE = 23;
+ protected final static int STATE_LONG_SHARED = 24;
+ protected final static int STATE_RAW_BINARY = 25;
+ protected final static int STATE_QUOTED_BINARY = 26;
+
+ /*
+ /**********************************************************************
+ /* Configuration
+ /**********************************************************************
+ */
+
+ /**
+ * Codec used for data binding when (if) requested.
+ */
+ protected ObjectCodec _objectCodec;
+
+ /**
+ * Flag that indicates whether content can legally have raw (unquoted)
+ * binary data. Since this information is included both in header and
+ * in actual binary data blocks there is redundancy, and we want to
+ * ensure settings are compliant. Using application may also want to
+ * know this setting in case it does some direct (random) access.
+ */
+ protected boolean _mayContainRawBinary;
+
+ protected final boolean _cfgRequireHeader;
+
+ /**
+ * Helper object used for low-level recycling of Smile-generator
+ * specific buffers.
+ */
+ final protected SmileBufferRecycler<String> _smileBufferRecycler;
+
+ /*
+ /**********************************************************************
+ /* Input source config
+ /**********************************************************************
+ */
+
+ /**
+ * This buffer is actually provided via {@link NonBlockingInputFeeder}
+ */
+ protected byte[] _inputBuffer = NO_BYTES;
+
+ /**
+ * In addition to current buffer pointer, and end pointer,
+ * we will also need to know number of bytes originally
+ * contained. This is needed to correctly update location
+ * information when the block has been completed.
+ */
+ protected int _origBufferLen;
+
+ // And from ParserBase:
+// protected int _inputPtr;
+// protected int _inputEnd;
+
+ /*
+ /**********************************************************************
+ /* Additional parsing state
+ /**********************************************************************
+ */
+
+ /**
+ * Current main decoding state
+ */
+ protected int _state;
+
+ /**
+ * Addition indicator within state; contextually relevant for just that state
+ */
+ protected int _substate;
+
+ /**
+ * Flag that indicates that the current token has not yet
+ * been fully processed, and needs to be finished for
+ * some access (or skipped to obtain the next token)
+ */
+ protected boolean _tokenIncomplete;
+
+ /**
+ * Specific flag that is set when we encountered a 32-bit
+ * floating point value; needed since numeric super classes do
+ * not track distinction between float and double, but Smile
+ * format does, and we want to retain that separation.
+ */
+ protected boolean _got32BitFloat;
+
+ /**
+ * For 32-bit values, we may use this for combining values
+ */
+ protected int _pendingInt;
+
+ /**
+ * For 64-bit values, we may use this for combining values
+ */
+ protected long _pendingLong;
+
+ /**
+ * Flag that is sent when calling application indicates that there will
+ * be no more input to parse.
+ */
+ protected boolean _endOfInput = false;
+
+ /*
+ /**********************************************************************
+ /* Symbol handling, decoding
+ /**********************************************************************
+ */
+
+ /**
+ * Symbol table that contains field names encountered so far
+ */
+ final protected BytesToNameCanonicalizer _symbols;
+
+ /**
+ * Temporary buffer used for name parsing.
+ */
+ protected int[] _quadBuffer = NO_INTS;
+
+ /*
+ /**********************************************************************
+ /* Name/entity parsing state
+ /**********************************************************************
+ */
+
+ /**
+ * Number of complete quads parsed for current name (quads
+ * themselves are stored in {@link #_quadBuffer}).
+ */
+ protected int _quadCount;
+
+ /**
+ * Bytes parsed for the current, incomplete, quad
+ */
+ protected int _currQuad;
+
+ /**
+ * Number of bytes pending/buffered, stored in {@link #_currQuad}
+ */
+ protected int _currQuadBytes = 0;
+
+ /**
+ * Array of recently seen field names, which may be back referenced
+ * by later fields.
+ * Defaults set to enable handling even if no header found.
+ */
+ protected String[] _seenNames = NO_STRINGS;
+
+ protected int _seenNameCount = 0;
+
+ /**
+ * Array of recently seen field names, which may be back referenced
+ * by later fields
+ * Defaults set to disable handling if no header found.
+ */
+ protected String[] _seenStringValues = null;
+
+ protected int _seenStringValueCount = -1;
+
+ /*
+ /**********************************************************************
+ /* Thread-local recycling
+ /**********************************************************************
+ */
+
+ /**
+ * <code>ThreadLocal</code> contains a {@link java.lang.ref.SoftReference}
+ * to a buffer recycler used to provide a low-cost
+ * buffer recycling for Smile-specific buffers.
+ */
+ final protected static ThreadLocal<SoftReference<SmileBufferRecycler<String>>> _smileRecyclerRef
+ = new ThreadLocal<SoftReference<SmileBufferRecycler<String>>>();
+
+ /*
+ /**********************************************************************
+ /* Life-cycle
+ /**********************************************************************
+ */
+
+ public NonBlockingParserImpl(IOContext ctxt, int parserFeatures, int smileFeatures,
+ ObjectCodec codec, BytesToNameCanonicalizer sym)
+ {
+ super(ctxt, parserFeatures);
+ _objectCodec = codec;
+ _symbols = sym;
+
+ _tokenInputRow = -1;
+ _tokenInputCol = -1;
+ _smileBufferRecycler = _smileBufferRecycler();
+
+ _currToken = JsonToken.NOT_AVAILABLE;
+ _state = STATE_INITIAL;
+ _tokenIncomplete = true;
+
+ _cfgRequireHeader = (smileFeatures & SmileParser.Feature.REQUIRE_HEADER.getMask()) != 0;
+ }
+
+ @Override
+ public ObjectCodec getCodec() {
+ return _objectCodec;
+ }
+
+ @Override
+ public void setCodec(ObjectCodec c) {
+ _objectCodec = c;
+ }
+
+ /**
+ * Helper method called when it looks like input might contain the signature;
+ * and it is necessary to detect and handle signature to get configuration
+ * information it might have.
+ *
+ * @return True if valid signature was found and handled; false if not
+ */
+ protected boolean handleSignature(boolean consumeFirstByte, boolean throwException)
+ throws IOException, JsonParseException
+ {
+ if (consumeFirstByte) {
+ ++_inputPtr;
+ }
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ if (_inputBuffer[_inputPtr] != SmileConstants.HEADER_BYTE_2) {
+ if (throwException) {
+ _reportError("Malformed content: signature not valid, starts with 0x3a but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr])+", not 0x29");
+ }
+ return false;
+ }
+ if (++_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ if (_inputBuffer[_inputPtr] != SmileConstants.HEADER_BYTE_3) {
+ if (throwException) {
+ _reportError("Malformed content: signature not valid, starts with 0x3a, 0x29, but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr])+", not 0xA");
+ }
+ return false;
+ }
+ // Good enough; just need version info from 4th byte...
+ if (++_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ int versionBits = (ch >> 4) & 0x0F;
+ // but failure with version number is fatal, can not ignore
+ if (versionBits != SmileConstants.HEADER_VERSION_0) {
+ _reportError("Header version number bits (0x"+Integer.toHexString(versionBits)+") indicate unrecognized version; only 0x0 handled by parser");
+ }
+
+ // can avoid tracking names, if explicitly disabled
+ if ((ch & SmileConstants.HEADER_BIT_HAS_SHARED_NAMES) == 0) {
+ _seenNames = null;
+ _seenNameCount = -1;
+ }
+ // conversely, shared string values must be explicitly enabled
+ if ((ch & SmileConstants.HEADER_BIT_HAS_SHARED_STRING_VALUES) != 0) {
+ _seenStringValues = NO_STRINGS;
+ _seenStringValueCount = 0;
+ }
+ _mayContainRawBinary = ((ch & SmileConstants.HEADER_BIT_HAS_RAW_BINARY) != 0);
+ return true;
+ }
+
+ protected final static SmileBufferRecycler<String> _smileBufferRecycler()
+ {
+ SoftReference<SmileBufferRecycler<String>> ref = _smileRecyclerRef.get();
+ SmileBufferRecycler<String> br = (ref == null) ? null : ref.get();
+
+ if (br == null) {
+ br = new SmileBufferRecycler<String>();
+ _smileRecyclerRef.set(new SoftReference<SmileBufferRecycler<String>>(br));
+ }
+ return br;
+ }
+
+ /*
+ /**********************************************************************
+ /* Versioned
+ /**********************************************************************
+ */
+
+ @Override
+ public Version version() {
+ return PackageVersion.VERSION;
+ }
+
+ /*
+ /**********************************************************************
+ /* Former StreamBasedParserBase methods
+ /**********************************************************************
+ */
+
+ @Override
+ public int releaseBuffered(OutputStream out) throws IOException
+ {
+ int count = _inputEnd - _inputPtr;
+ if (count < 1) {
+ return 0;
+ }
+ // let's just advance ptr to end
+ int origPtr = _inputPtr;
+ out.write(_inputBuffer, origPtr, count);
+ return count;
+ }
+
+ @Override
+ public Object getInputSource() {
+ // since input is "pushed", to traditional source...
+ return null;
+ }
+
+ /**
+ * Overridden since we do not really have character-based locations,
+ * but we do have byte offset to specify.
+ */
+ @Override
+ public JsonLocation getTokenLocation()
+ {
+ // token location is correctly managed...
+ return new JsonLocation(_ioContext.getSourceReference(),
+ _tokenInputTotal, // bytes
+ -1, -1, (int) _tokenInputTotal); // char offset, line, column
+ }
+
+ /**
+ * Overridden since we do not really have character-based locations,
+ * but we do have byte offset to specify.
+ */
+ @Override
+ public JsonLocation getCurrentLocation()
+ {
+ final long offset = _currInputProcessed + _inputPtr;
+ return new JsonLocation(_ioContext.getSourceReference(),
+ offset, // bytes
+ -1, -1, (int) offset); // char offset, line, column
+ }
+
+ /*
+ /**********************************************************************
+ /* Low-level reading, other
+ /**********************************************************************
+ */
+
+ @Override
+ protected final boolean loadMore() throws IOException {
+ _throwInternal();
+ return false;
+ }
+
+ /**
+ * Helper method that will try to load at least specified number bytes in
+ * input buffer, possible moving existing data around if necessary
+ */
+ protected final boolean _loadToHaveAtLeast(int minAvailable) throws IOException
+ {
+ _throwInternal();
+ return false;
+ }
+
+ @Override
+ protected void _closeInput() throws IOException {
+ // nothing to do here
+ }
+
+ /*
+ /**********************************************************************
+ /* Overridden methods
+ /**********************************************************************
+ */
+
+ @Override
+ protected void _finishString() throws IOException, JsonParseException {
+ // should never be called; but must be defined for superclass
+ _throwInternal();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ super.close();
+ // Merge found symbols, if any:
+ _symbols.release();
+ }
+
+ @Override
+ public boolean hasTextCharacters()
+ {
+ if (_currToken == JsonToken.VALUE_STRING) {
+ // yes; is or can be made available efficiently as char[]
+ return _textBuffer.hasTextAsCharacters();
+ }
+ if (_currToken == JsonToken.FIELD_NAME) {
+ // not necessarily; possible but:
+ return _nameCopied;
+ }
+ // other types, no benefit from accessing as char[]
+ return false;
+ }
+
+ /**
+ * Method called to release internal buffers owned by the base
+ * reader. This may be called along with {@link #_closeInput} (for
+ * example, when explicitly closing this reader instance), or
+ * separately (if need be).
+ */
+ @Override
+ protected void _releaseBuffers() throws IOException
+ {
+ super._releaseBuffers();
+ {
+ String[] nameBuf = _seenNames;
+ if (nameBuf != null && nameBuf.length > 0) {
+ _seenNames = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
+ * but we only need to clear up to count as it is not a hash area
+ */
+ if (_seenNameCount > 0) {
+ Arrays.fill(nameBuf, 0, _seenNameCount, null);
+ }
+ _smileBufferRecycler.releaseSeenNamesBuffer(nameBuf);
+ }
+ }
+ {
+ String[] valueBuf = _seenStringValues;
+ if (valueBuf != null && valueBuf.length > 0) {
+ _seenStringValues = null;
+ /* 28-Jun-2011, tatu: With 1.9, caller needs to clear the buffer;
+ * but we only need to clear up to count as it is not a hash area
+ */
+ if (_seenStringValueCount > 0) {
+ Arrays.fill(valueBuf, 0, _seenStringValueCount, null);
+ }
+ _smileBufferRecycler.releaseSeenStringValuesBuffer(valueBuf);
+ }
+ }
+ }
+
+ /*
+ /**********************************************************************
+ /* Extended API
+ /**********************************************************************
+ */
+
+ public boolean mayContainRawBinary() {
+ return _mayContainRawBinary;
+ }
+
+ /*
+ /**********************************************************************
+ /* JsonParser impl
+ /**********************************************************************
+ */
+
+ @Override
+ public JsonToken nextToken() throws IOException, JsonParseException
+ {
+ _numTypesValid = NR_UNKNOWN;
+ // have we already decoded part of event? If so, continue...
+ if (_tokenIncomplete) {
+ // we might be able to optimize by separate skipping, but for now:
+ return _finishToken();
+ }
+ _tokenInputTotal = _currInputProcessed + _inputPtr;
+ // also: clear any data retained so far
+ _binaryValue = null;
+ // Two main modes: values, and field names.
+ if (_parsingContext.inObject() && _currToken != JsonToken.FIELD_NAME) {
+ return (_currToken = _handleFieldName());
+ }
+ if (_inputPtr >= _inputEnd) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ switch ((ch >> 5) & 0x7) {
+ case 0: // short shared string value reference
+ if (ch == 0) { // important: this is invalid, don't accept
+ _reportError("Invalid token byte 0x00");
+ }
+ return _handleSharedString(ch-1);
+
+ case 1: // simple literals, numbers
+ {
+ _numTypesValid = 0;
+ switch (ch & 0x1F) {
+ case 0x00:
+ _textBuffer.resetWithEmpty();
+ return (_currToken = JsonToken.VALUE_STRING);
+ case 0x01:
+ return (_currToken = JsonToken.VALUE_NULL);
+ case 0x02: // false
+ return (_currToken = JsonToken.VALUE_FALSE);
+ case 0x03: // 0x03 == true
+ return (_currToken = JsonToken.VALUE_TRUE);
+ case 0x04:
+ _state = STATE_NUMBER_INT;
+ return _nextInt(0, 0);
+ case 0x05:
+ _numberLong = 0;
+ _state = STATE_NUMBER_LONG;
+ return _nextLong(0, 0L);
+ case 0x06:
+ _state = STATE_NUMBER_BIGINT;
+ return _nextBigInt(0);
+ case 0x07: // illegal
+ break;
+ case 0x08:
+ _pendingInt = 0;
+ _state = STATE_NUMBER_FLOAT;
+ _got32BitFloat = true;
+ return _nextFloat(0, 0);
+ case 0x09:
+ _pendingLong = 0L;
+ _state = STATE_NUMBER_DOUBLE;
+ _got32BitFloat = false;
+ return _nextDouble(0, 0L);
+ case 0x0A:
+ _state = STATE_NUMBER_BIGDEC;
+ return _nextBigDecimal(0);
+ case 0x0B: // illegal
+ break;
+ case 0x1A: // == 0x3A == ':' -> possibly header signature for next chunk?
+ if (!_handleHeader(0)) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ //if (handleSignature(false, false)) {
+ /* Ok, now; end-marker and header both imply doc boundary and a
+ * 'null token'; but if both are seen, they are collapsed.
+ * We can check this by looking at current token; if it's null,
+ * need to get non-null token
+ */
+ if (_currToken == null) {
+ return nextToken();
+ }
+ return (_currToken = null);
+ }
+ }
+ // and everything else is reserved, for now
+ break;
+ case 2: // tiny ASCII
+ // fall through
+ case 3: // short ASCII
+ // fall through
+ return _nextShortAscii(0);
+
+ case 4: // tiny Unicode
+ // fall through
+ case 5: // short Unicode
+ // No need to decode, unless we have to keep track of back-references (for shared string values)
+ _currToken = JsonToken.VALUE_STRING;
+ if (_seenStringValueCount >= 0) { // shared text values enabled
+ _addSeenStringValue();
+ } else {
+ _tokenIncomplete = true;
+ }
+ return _nextShortUnicode(0);
+
+ case 6: // small integers; zigzag encoded
+ _numberInt = SmileUtil.zigzagDecode(ch & 0x1F);
+ _numTypesValid = NR_INT;
+ return (_currToken = JsonToken.VALUE_NUMBER_INT);
+ case 7: // binary/long-text/long-shared/start-end-markers
+ switch (ch & 0x1F) {
+ case 0x00: // long variable length ASCII
+ return _nextLongAscii(0);
+ case 0x04: // long variable length unicode
+ return _nextLongUnicode(0);
+ case 0x08: // binary, 7-bit
+ return _nextQuotedBinary(0);
+ case 0x0C: // long shared string
+ case 0x0D:
+ case 0x0E:
+ case 0x0F:
+ return _nextLongSharedString(0);
+// return _handleSharedString(((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF));
+ case 0x18: // START_ARRAY
+ _parsingContext = _parsingContext.createChildArrayContext(-1, -1);
+ return (_currToken = JsonToken.START_ARRAY);
+ case 0x19: // END_ARRAY
+ if (!_parsingContext.inArray()) {
+ _reportMismatchedEndMarker(']', '}');
+ }
+ _parsingContext = _parsingContext.getParent();
+ return (_currToken = JsonToken.END_ARRAY);
+ case 0x1A: // START_OBJECT
+ _parsingContext = _parsingContext.createChildObjectContext(-1, -1);
+ return (_currToken = JsonToken.START_OBJECT);
+ case 0x1B: // not used in this mode; would be END_OBJECT
+ _reportError("Invalid type marker byte 0xFB in value mode (would be END_OBJECT in key mode)");
+ case 0x1D: // binary, raw
+ // should we validate this is legal? (as per header)
+ return _nextRawBinary(0);
+ case 0x1F: // 0xFF, end of content
+ return (_currToken = null);
+ }
+ break;
+ }
+ // If we get this far, type byte is corrupt
+ _reportError("Invalid type marker byte 0x"+Integer.toHexString(ch & 0xFF)+" for expected value token");
+ return null;
+ }
+
+ private final JsonToken _handleSharedString(int index)
+ throws IOException, JsonParseException
+ {
+ if (index >= _seenStringValueCount) {
+ _reportInvalidSharedStringValue(index);
+ }
+ _textBuffer.resetWithString(_seenStringValues[index]);
+ return (_currToken = JsonToken.VALUE_STRING);
+ }
+
+ private final void _addSeenStringValue()
+ throws IOException, JsonParseException
+ {
+ _finishToken();
+ if (_seenStringValueCount < _seenStringValues.length) {
+ // !!! TODO: actually only store char[], first time around?
+ _seenStringValues[_seenStringValueCount++] = _textBuffer.contentsAsString();
+ return;
+ }
+ _expandSeenStringValues();
+ }
+
+ private final void _expandSeenStringValues()
+ {
+ String[] oldShared = _seenStringValues;
+ int len = oldShared.length;
+ String[] newShared;
+ if (len == 0) {
+ newShared = _smileBufferRecycler.allocSeenStringValuesBuffer();
+ if (newShared == null) {
+ newShared = new String[SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH];
+ }
+ } else if (len == SmileConstants.MAX_SHARED_STRING_VALUES) { // too many? Just flush...
+ newShared = oldShared;
+ _seenStringValueCount = 0; // could also clear, but let's not yet bother
+ } else {
+ int newSize = (len == SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH) ? 256 : SmileConstants.MAX_SHARED_STRING_VALUES;
+ newShared = new String[newSize];
+ System.arraycopy(oldShared, 0, newShared, 0, oldShared.length);
+ }
+ _seenStringValues = newShared;
+ _seenStringValues[_seenStringValueCount++] = _textBuffer.contentsAsString();
+ }
+
+ @Override
+ public String getCurrentName() throws IOException, JsonParseException
+ {
+ return _parsingContext.getCurrentName();
+ }
+
+ @Override
+ public NumberType getNumberType()
+ throws IOException, JsonParseException
+ {
+ if (_got32BitFloat) {
+ return NumberType.FLOAT;
+ }
+ return super.getNumberType();
+ }
+
+ /*
+ /**********************************************************************
+ /* AsyncInputFeeder impl
+ /**********************************************************************
+ */
+
+ public final boolean needMoreInput() {
+ return (_inputPtr >=_inputEnd) && !_endOfInput;
+ }
+
+ public void feedInput(byte[] buf, int start, int len)
+ throws IOException
+ {
+ // Must not have remaining input
+ if (_inputPtr < _inputEnd) {
+ throw new IOException("Still have "+(_inputEnd - _inputPtr)+" undecoded bytes, should not call 'feedInput'");
+ }
+ // and shouldn't have been marked as end-of-input
+ if (_endOfInput) {
+ throw new IOException("Already closed, can not feed more input");
+ }
+ // Time to update pointers first
+ _currInputProcessed += _origBufferLen;
+ _currInputRowStart -= _origBufferLen;
+
+ // And then update buffer settings
+ _inputBuffer = buf;
+ _inputPtr = start;
+ _inputEnd = start+len;
+ _origBufferLen = len;
+ }
+
+ public void endOfInput() {
+ _endOfInput = true;
+ }
+
+ /*
+ /**********************************************************************
+ /* NonBlockParser impl (except for NonBlockingInputFeeder)
+ /**********************************************************************
+ */
+
+ public JsonToken peekNextToken() throws IOException, JsonParseException
+ {
+ if (!_tokenIncomplete) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ switch (_state) {
+ case STATE_INITIAL: // the case if no input has yet been fed
+ return JsonToken.NOT_AVAILABLE;
+ case STATE_HEADER:
+ return JsonToken.NOT_AVAILABLE;
+ case STATE_NUMBER_INT:
+ case STATE_NUMBER_LONG:
+ return JsonToken.VALUE_NUMBER_INT;
+ case STATE_NUMBER_FLOAT:
+ case STATE_NUMBER_DOUBLE:
+ case STATE_NUMBER_BIGDEC:
+ return JsonToken.VALUE_NUMBER_FLOAT;
+ }
+ throw new IllegalStateException("Internal error: unknown 'state', "+_state);
+ }
+
+ /*
+ /**********************************************************************
+ /* Internal methods: second-level parsing:
+ /**********************************************************************
+ */
+
+ private final JsonToken _nextInt(int substate, int value)
+ throws IOException, JsonParseException
+ {
+ while (_inputPtr < _inputEnd) {
+ int b = _inputBuffer[_inputPtr++];
+ if (b < 0) { // got it all; these are last 6 bits
+ value = (value << 6) | (b & 0x3F);
+ _numberInt = SmileUtil.zigzagDecode(value);
+ _numTypesValid = NR_INT;
+ _tokenIncomplete = false;
+ return (_currToken = JsonToken.VALUE_NUMBER_INT);
+ }
+ // can't get too big; 5 bytes is max
+ if (++substate >= 5 ) {
+ _reportError("Corrupt input; 32-bit VInt extends beyond 5 data bytes");
+ }
+ value = (value << 7) | b;
+ }
+ // did not get it all; mark the state so we know where to return:
+ _tokenIncomplete = true;
+ _substate = substate;
+ _pendingInt = value;
+ _state = STATE_NUMBER_INT;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextLong(int substate, long value) throws IOException, JsonParseException
+ {
+ while (_inputPtr < _inputEnd) {
+ int b = _inputBuffer[_inputPtr++];
+ if (b < 0) { // got it all; these are last 6 bits
+ value = (value << 6) | (b & 0x3F);
+ _numberLong = SmileUtil.zigzagDecode(value);
+ _numTypesValid = NR_LONG;
+ _tokenIncomplete = false;
+ return (_currToken = JsonToken.VALUE_NUMBER_INT);
+ }
+ // can't get too big; 10 bytes is max
+ if (++substate >= 10) {
+ _reportError("Corrupt input; 64-bit VInt extends beyond 10 data bytes");
+ }
+ value = (value << 7) | b;
+ }
+ // did not get it all; mark the state so we know where to return:
+ _tokenIncomplete = true;
+ _substate = substate;
+ _pendingLong = value;
+ _state = STATE_NUMBER_LONG;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextBigInt(int substate) throws IOException, JsonParseException
+ {
+ // !!! TBI
+ _tokenIncomplete = true;
+ _substate = substate;
+// _pendingLong = value;
+ _state = STATE_NUMBER_BIGDEC;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ /*
+ private final boolean _finishBigInteger()
+ throws IOException, JsonParseException
+ {
+ byte[] raw = _read7BitBinaryWithLength();
+ if (raw == null) {
+ return false;
+ }
+ _numberBigInt = new BigInteger(raw);
+ _numTypesValid = NR_BIGINT;
+ return true;
+ }
+*/
+
+ private final JsonToken _nextFloat(int substate, int value) throws IOException, JsonParseException
+ {
+ while (_inputPtr < _inputEnd) {
+ int b = _inputBuffer[_inputPtr++];
+ value = (value << 7) + b;
+ if (++substate == 5) { // done!
+ _numberDouble = (double) Float.intBitsToFloat(value);
+ _numTypesValid = NR_DOUBLE;
+ _tokenIncomplete = false;
+ return (_currToken = JsonToken.VALUE_NUMBER_FLOAT);
+ }
+ }
+ _tokenIncomplete = true;
+ _substate = substate;
+ _pendingInt = value;
+ _state = STATE_NUMBER_FLOAT;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextDouble(int substate, long value) throws IOException, JsonParseException
+ {
+ while (_inputPtr < _inputEnd) {
+ int b = _inputBuffer[_inputPtr++];
+ value = (value << 7) + b;
+ if (++substate == 10) { // done!
+ _numberDouble = Double.longBitsToDouble(value);
+ _numTypesValid = NR_DOUBLE;
+ _tokenIncomplete = false;
+ return (_currToken = JsonToken.VALUE_NUMBER_FLOAT);
+ }
+ }
+ _tokenIncomplete = true;
+ _substate = substate;
+ _pendingLong = value;
+ _state = STATE_NUMBER_DOUBLE;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextBigDecimal(int substate) throws IOException, JsonParseException
+ {
+ // !!! TBI
+ _tokenIncomplete = true;
+ _substate = substate;
+// _pendingLong = value;
+ _state = STATE_NUMBER_BIGDEC;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+/*
+ private final void _finishBigDecimal()
+ throws IOException, JsonParseException
+ {
+ int scale = SmileUtil.zigzagDecode(_readUnsignedVInt());
+ byte[] raw = _read7BitBinaryWithLength();
+ _numberBigDecimal = new BigDecimal(new BigInteger(raw), scale);
+ _numTypesValid = NR_BIGDECIMAL;
+ }
+
+ */
+
+ /*
+ private final int _readUnsignedVInt()
+ throws IOException, JsonParseException
+ {
+ int value = 0;
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int i = _inputBuffer[_inputPtr++];
+ if (i < 0) { // last byte
+ value = (value << 6) + (i & 0x3F);
+ return value;
+ }
+ value = (value << 7) + i;
+ }
+ }
+ */
+
+ private final boolean _handleHeader(int substate) throws IOException, JsonParseException
+ {
+ while (_inputPtr < _inputEnd) {
+ byte b = _inputBuffer[_inputPtr++];
+ switch (substate) {
+ case 0: // after first byte
+ if (b != SmileConstants.HEADER_BYTE_2) {
+ _reportError("Malformed content: header signature not valid, starts with 0x3a but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr] & 0xFF)+", not 0x29");
+ }
+ break;
+ case 1:
+ if (b != SmileConstants.HEADER_BYTE_3) {
+ _reportError("Malformed content: signature not valid, starts with 0x3a, 0x29, but followed by 0x"
+ +Integer.toHexString(_inputBuffer[_inputPtr & 0xFF])+", not 0x0A");
+ }
+ break;
+ case 2: // ok, here be the version, config bits...
+ int versionBits = (b >> 4) & 0x0F;
+ // but failure with version number is fatal, can not ignore
+ if (versionBits != SmileConstants.HEADER_VERSION_0) {
+ _reportError("Header version number bits (0x"+Integer.toHexString(versionBits)+") indicate unrecognized version; only 0x0 handled by parser");
+ }
+
+ // can avoid tracking names, if explicitly disabled
+ if ((b & SmileConstants.HEADER_BIT_HAS_SHARED_NAMES) == 0) {
+ _seenNames = null;
+ _seenNameCount = -1;
+ }
+ // conversely, shared string values must be explicitly enabled
+ if ((b & SmileConstants.HEADER_BIT_HAS_SHARED_STRING_VALUES) != 0) {
+ _seenStringValues = NO_STRINGS;
+ _seenStringValueCount = 0;
+ }
+ _mayContainRawBinary = ((b & SmileConstants.HEADER_BIT_HAS_RAW_BINARY) != 0);
+ _tokenIncomplete = false;
+ return true;
+ }
+ }
+ _tokenIncomplete = true;
+ _state = STATE_HEADER;
+ _substate = substate;
+ return false;
+ }
+
+ private final JsonToken _nextShortAscii(int substate) throws IOException, JsonParseException
+ {
+ _state = STATE_SHORT_ASCII;
+ _tokenIncomplete = true;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextShortUnicode(int substate) throws IOException, JsonParseException
+ {
+ _state = STATE_SHORT_UNICODE;
+ _tokenIncomplete = true;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ /*
+ protected final void _decodeShortAsciiValue(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ // Note: we count on fact that buffer must have at least 'len' (<= 64) empty char slots
+ final char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int outPtr = 0;
+ final byte[] inBuf = _inputBuffer;
+ int inPtr = _inputPtr;
+
+ // meaning: regular tight loop is no slower, typically faster here:
+ for (final int end = inPtr + len; inPtr < end; ++inPtr) {
+ outBuf[outPtr++] = (char) inBuf[inPtr];
+ }
+
+ _inputPtr = inPtr;
+ _textBuffer.setCurrentLength(len);
+ }
+
+ protected final void _decodeShortUnicodeValue(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int inPtr = _inputPtr;
+ _inputPtr += len;
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ final byte[] inputBuf = _inputBuffer;
+ for (int end = inPtr + len; inPtr < end; ) {
+ int i = inputBuf[inPtr++] & 0xFF;
+ int code = codes[i];
+ if (code != 0) {
+ // trickiest one, need surrogate handling
+ switch (code) {
+ case 1:
+ i = ((i & 0x1F) << 6) | (inputBuf[inPtr++] & 0x3F);
+ break;
+ case 2:
+ i = ((i & 0x0F) << 12)
+ | ((inputBuf[inPtr++] & 0x3F) << 6)
+ | (inputBuf[inPtr++] & 0x3F);
+ break;
+ case 3:
+ i = ((i & 0x07) << 18)
+ | ((inputBuf[inPtr++] & 0x3F) << 12)
+ | ((inputBuf[inPtr++] & 0x3F) << 6)
+ | (inputBuf[inPtr++] & 0x3F);
+ // note: this is the codepoint value; need to split, too
+ i -= 0x10000;
+ outBuf[outPtr++] = (char) (0xD800 | (i >> 10));
+ i = 0xDC00 | (i & 0x3FF);
+ break;
+ default: // invalid
+ _reportError("Invalid byte "+Integer.toHexString(i)+" in short Unicode text block");
+ }
+ }
+ outBuf[outPtr++] = (char) i;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+ */
+
+ private final JsonToken _nextLongAscii(int substate) throws IOException, JsonParseException
+ {
+ // did not get it all; mark the state so we know where to return:
+ _state = STATE_LONG_ASCII;
+ _tokenIncomplete = true;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ /*
+ private final void _decodeLongAscii()
+ throws IOException, JsonParseException
+ {
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ main_loop:
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int inPtr = _inputPtr;
+ int left = _inputEnd - inPtr;
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ left = Math.min(left, outBuf.length - outPtr);
+ do {
+ byte b = _inputBuffer[inPtr++];
+ if (b == SmileConstants.BYTE_MARKER_END_OF_STRING) {
+ _inputPtr = inPtr;
+ break main_loop;
+ }
+ outBuf[outPtr++] = (char) b;
+ } while (--left > 0);
+ _inputPtr = inPtr;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+ */
+
+ private final JsonToken _nextLongUnicode(int substate) throws IOException, JsonParseException
+ {
+ // did not get it all; mark the state so we know where to return:
+ _state = STATE_LONG_UNICODE;
+ _tokenIncomplete = true;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+
+/*
+ private final void _decodeLongUnicode()
+ throws IOException, JsonParseException
+ {
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ int c;
+ final byte[] inputBuffer = _inputBuffer;
+
+ main_loop:
+ while (true) {
+ // First the tight ASCII loop:
+ ascii_loop:
+ while (true) {
+ int ptr = _inputPtr;
+ if (ptr >= _inputEnd) {
+ loadMoreGuaranteed();
+ ptr = _inputPtr;
+ }
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ int max = _inputEnd;
+ {
+ int max2 = ptr + (outBuf.length - outPtr);
+ if (max2 < max) {
+ max = max2;
+ }
+ }
+ while (ptr < max) {
+ c = (int) inputBuffer[ptr++] & 0xFF;
+ if (codes[c] != 0) {
+ _inputPtr = ptr;
+ break ascii_loop;
+ }
+ outBuf[outPtr++] = (char) c;
+ }
+ _inputPtr = ptr;
+ }
+ // Ok: end marker, escape or multi-byte?
+ if (c == SmileConstants.INT_MARKER_END_OF_STRING) {
+ break main_loop;
+ }
+
+ switch (codes[c]) {
+ case 1: // 2-byte UTF
+ c = _decodeUtf8_2(c);
+ break;
+ case 2: // 3-byte UTF
+ if ((_inputEnd - _inputPtr) >= 2) {
+ c = _decodeUtf8_3fast(c);
+ } else {
+ c = _decodeUtf8_3(c);
+ }
+ break;
+ case 3: // 4-byte UTF
+ c = _decodeUtf8_4(c);
+ // Let's add first part right away:
+ outBuf[outPtr++] = (char) (0xD800 | (c >> 10));
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ c = 0xDC00 | (c & 0x3FF);
+ // And let the other char output down below
+ break;
+ default:
+ // Is this good enough error message?
+ _reportInvalidChar(c);
+ }
+ // Need more room?
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ // Ok, let's add char to output:
+ outBuf[outPtr++] = (char) c;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ }
+ */
+
+ private final JsonToken _nextLongSharedString(int substate) throws IOException, JsonParseException
+ {
+ // did not get it all; mark the state so we know where to return:
+ _tokenIncomplete = true;
+ _state = STATE_LONG_SHARED;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ private final JsonToken _nextRawBinary(int substate) throws IOException, JsonParseException
+ {
+ // did not get it all; mark the state so we know where to return:
+ _tokenIncomplete = true;
+ _state = STATE_RAW_BINARY;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+/*
+
+ private final void _finishRawBinary()
+ throws IOException, JsonParseException
+ {
+ int byteLen = _readUnsignedVInt();
+ _binaryValue = new byte[byteLen];
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ptr = 0;
+ while (true) {
+ int toAdd = Math.min(byteLen, _inputEnd - _inputPtr);
+ System.arraycopy(_inputBuffer, _inputPtr, _binaryValue, ptr, toAdd);
+ _inputPtr += toAdd;
+ ptr += toAdd;
+ byteLen -= toAdd;
+ if (byteLen <= 0) {
+ return;
+ }
+ loadMoreGuaranteed();
+ }
+ }
+ */
+
+ private final JsonToken _nextQuotedBinary(int substate) throws IOException, JsonParseException
+ {
+ // did not get it all; mark the state so we know where to return:
+ _tokenIncomplete = true;
+ _state = STATE_QUOTED_BINARY;
+ _substate = substate;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+
+ /*
+ private final byte[] _read7BitBinaryWithLength()
+ throws IOException, JsonParseException
+ {
+ int byteLen = _readUnsignedVInt();
+ byte[] result = new byte[byteLen];
+ int ptr = 0;
+ int lastOkPtr = byteLen - 7;
+
+ // first, read all 7-by-8 byte chunks
+ while (ptr <= lastOkPtr) {
+ if ((_inputEnd - _inputPtr) < 8) {
+ _loadToHaveAtLeast(8);
+ }
+ int i1 = (_inputBuffer[_inputPtr++] << 25)
+ + (_inputBuffer[_inputPtr++] << 18)
+ + (_inputBuffer[_inputPtr++] << 11)
+ + (_inputBuffer[_inputPtr++] << 4);
+ int x = _inputBuffer[_inputPtr++];
+ i1 += x >> 3;
+ int i2 = ((x & 0x7) << 21)
+ + (_inputBuffer[_inputPtr++] << 14)
+ + (_inputBuffer[_inputPtr++] << 7)
+ + _inputBuffer[_inputPtr++];
+ // Ok: got our 7 bytes, just need to split, copy
+ result[ptr++] = (byte)(i1 >> 24);
+ result[ptr++] = (byte)(i1 >> 16);
+ result[ptr++] = (byte)(i1 >> 8);
+ result[ptr++] = (byte)i1;
+ result[ptr++] = (byte)(i2 >> 16);
+ result[ptr++] = (byte)(i2 >> 8);
+ result[ptr++] = (byte)i2;
+ }
+ // and then leftovers: n+1 bytes to decode n bytes
+ int toDecode = (result.length - ptr);
+ if (toDecode > 0) {
+ if ((_inputEnd - _inputPtr) < (toDecode+1)) {
+ _loadToHaveAtLeast(toDecode+1);
+ }
+ int value = _inputBuffer[_inputPtr++];
+ for (int i = 1; i < toDecode; ++i) {
+ value = (value << 7) + _inputBuffer[_inputPtr++];
+ result[ptr++] = (byte) (value >> (7 - i));
+ }
+ // last byte is different, has remaining 1 - 6 bits, right-aligned
+ value <<= toDecode;
+ result[ptr] = (byte) (value + _inputBuffer[_inputPtr++]);
+ }
+ return result;
+ }
+ */
+
+ /*
+ /**********************************************************************
+ /* Public API, traversal, nextXxxValue/nextFieldName
+ /**********************************************************************
+ */
+
+ /* Implementing these methods efficiently for non-blocking cases would
+ * be complicated; so for now let's just use the default non-optimized
+ * implementation
+ */
+
+// public boolean nextFieldName(SerializableString str) throws IOException, JsonParseException
+// public String nextTextValue() throws IOException, JsonParseException
+// public int nextIntValue(int defaultValue) throws IOException, JsonParseException
+// public long nextLongValue(long defaultValue) throws IOException, JsonParseException
+// public Boolean nextBooleanValue() throws IOException, JsonParseException
+
+ /*
+ /**********************************************************************
+ /* Public API, access to token information, text
+ /**********************************************************************
+ */
+
+ /**
+ * Method for accessing textual representation of the current event;
+ * if no current event (before first call to {@link #nextToken}, or
+ * after encountering end-of-input), returns null.
+ * Method can be called for any event.
+ */
+ @Override
+ public String getText()
+ throws IOException, JsonParseException
+ {
+ if (_currToken == JsonToken.VALUE_STRING) {
+ return _textBuffer.contentsAsString();
+ }
+ if (_tokenIncomplete) {
+ return null;
+ }
+ JsonToken t = _currToken;
+ if (t == null) { // null only before/after document
+ return null;
+ }
+ if (t == JsonToken.FIELD_NAME) {
+ return _parsingContext.getCurrentName();
+ }
+ if (t.isNumeric()) {
+ // TODO: optimize?
+ return getNumberValue().toString();
+ }
+ return _currToken.asString();
+ }
+
+ @Override
+ public char[] getTextCharacters()
+ throws IOException, JsonParseException
+ {
+ if (_currToken != null) { // null only before/after document
+ switch (_currToken) {
+ case VALUE_STRING:
+ return _textBuffer.getTextBuffer();
+ case FIELD_NAME:
+ if (!_nameCopied) {
+ String name = _parsingContext.getCurrentName();
+ int nameLen = name.length();
+ if (_nameCopyBuffer == null) {
+ _nameCopyBuffer = _ioContext.allocNameCopyBuffer(nameLen);
+ } else if (_nameCopyBuffer.length < nameLen) {
+ _nameCopyBuffer = new char[nameLen];
+ }
+ name.getChars(0, nameLen, _nameCopyBuffer, 0);
+ _nameCopied = true;
+ }
+ return _nameCopyBuffer;
+
+ // fall through
+ case VALUE_NUMBER_INT:
+ case VALUE_NUMBER_FLOAT:
+ // TODO: optimize
+ return getNumberValue().toString().toCharArray();
+ default:
+ if (_tokenIncomplete) {
+ return null;
+ }
+ return _currToken.asCharArray();
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public int getTextLength()
+ throws IOException, JsonParseException
+ {
+ if (_currToken != null) { // null only before/after document
+ if (_tokenIncomplete) {
+ return -1; // or throw exception?
+ }
+ switch (_currToken) {
+ case VALUE_STRING:
+ return _textBuffer.size();
+ case FIELD_NAME:
+ return _parsingContext.getCurrentName().length();
+ // fall through
+ case VALUE_NUMBER_INT:
+ case VALUE_NUMBER_FLOAT:
+ // TODO: optimize
+ return getNumberValue().toString().length();
+
+ default:
+ return _currToken.asCharArray().length;
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ public int getTextOffset() throws IOException, JsonParseException
+ {
+ return 0;
+ }
+
+ /*
+ /**********************************************************************
+ /* Public API, access to token information, binary
+ /**********************************************************************
+ */
+
+ @Override
+ public byte[] getBinaryValue(Base64Variant b64variant)
+ throws IOException, JsonParseException
+ {
+ if (_currToken != JsonToken.VALUE_EMBEDDED_OBJECT ) {
+ // Todo, maybe: support base64 for text?
+ _reportError("Current token ("+_currToken+") not VALUE_EMBEDDED_OBJECT, can not access as binary");
+ }
+ return _binaryValue;
+ }
+
+ @Override
+ public Object getEmbeddedObject()
+ throws IOException, JsonParseException
+ {
+ if (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT ) {
+ return _binaryValue;
+ }
+ return null;
+ }
+
+ // could possibly implement this... or maybe not.
+ @Override
+ public int readBinaryValue(Base64Variant b64variant, OutputStream out)
+ throws IOException, JsonParseException {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ /**********************************************************************
+ /* Internal methods, field name parsing
+ /**********************************************************************
+ */
+
+ /**
+ * Method that handles initial token type recognition for token
+ * that has to be either FIELD_NAME or END_OBJECT.
+ */
+ protected final JsonToken _handleFieldName() throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int ch = _inputBuffer[_inputPtr++];
+ switch ((ch >> 6) & 3) {
+ case 0: // misc, including end marker
+ switch (ch) {
+ case 0x20: // empty String as name, legal if unusual
+ _parsingContext.setCurrentName("");
+ return JsonToken.FIELD_NAME;
+ case 0x30: // long shared
+ case 0x31:
+ case 0x32:
+ case 0x33:
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int index = ((ch & 0x3) << 8) + (_inputBuffer[_inputPtr++] & 0xFF);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ _parsingContext.setCurrentName(_seenNames[index]);
+ }
+ return JsonToken.FIELD_NAME;
+ case 0x34: // long ASCII/Unicode name
+ _handleLongFieldName();
+ return JsonToken.FIELD_NAME;
+ }
+ break;
+ case 1: // short shared, can fully process
+ {
+ int index = (ch & 0x3F);
+ if (index >= _seenNameCount) {
+ _reportInvalidSharedName(index);
+ }
+ _parsingContext.setCurrentName(_seenNames[index]);
+ }
+ return JsonToken.FIELD_NAME;
+ case 2: // short ASCII
+ {
+ int len = 1 + (ch & 0x3f);
+ String name;
+ Name n = _findDecodedFromSymbols(len);
+ if (n != null) {
+ name = n.getName();
+ _inputPtr += len;
+ } else {
+ name = _decodeShortAsciiName(len);
+ name = _addDecodedToSymbols(len, name);
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ }
+ return JsonToken.FIELD_NAME;
+ case 3: // short Unicode
+ // all valid, except for 0xFF
+ ch &= 0x3F;
+ {
+ if (ch > 0x37) {
+ if (ch == 0x3B) {
+ if (!_parsingContext.inObject()) {
+ _reportMismatchedEndMarker('}', ']');
+ }
+ _parsingContext = _parsingContext.getParent();
+ return JsonToken.END_OBJECT;
+ }
+ } else {
+ final int len = ch + 2; // values from 2 to 57...
+ String name;
+ Name n = _findDecodedFromSymbols(len);
+ if (n != null) {
+ name = n.getName();
+ _inputPtr += len;
+ } else {
+ name = _decodeShortUnicodeName(len);
+ name = _addDecodedToSymbols(len, name);
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ return JsonToken.FIELD_NAME;
+ }
+ }
+ break;
+ }
+ // Other byte values are illegal
+ _reportError("Invalid type marker byte 0x"+Integer.toHexString(_inputBuffer[_inputPtr-1])
+ +" for expected field name (or END_OBJECT marker)");
+ return null;
+ }
+
+ /**
+ * Method called to try to expand shared name area to fit one more potentially
+ * shared String. If area is already at its biggest size, will just clear
+ * the area (by setting next-offset to 0)
+ */
+ private final String[] _expandSeenNames(String[] oldShared)
+ {
+ int len = oldShared.length;
+ String[] newShared;
+ if (len == 0) {
+ newShared = _smileBufferRecycler.allocSeenNamesBuffer();
+ if (newShared == null) {
+ newShared = new String[SmileBufferRecycler.DEFAULT_NAME_BUFFER_LENGTH];
+ }
+ } else if (len == SmileConstants.MAX_SHARED_NAMES) { // too many? Just flush...
+ newShared = oldShared;
+ _seenNameCount = 0; // could also clear, but let's not yet bother
+ } else {
+ int newSize = (len == SmileBufferRecycler.DEFAULT_STRING_VALUE_BUFFER_LENGTH) ? 256 : SmileConstants.MAX_SHARED_NAMES;
+ newShared = new String[newSize];
+ System.arraycopy(oldShared, 0, newShared, 0, oldShared.length);
+ }
+ return newShared;
+ }
+
+ private int _quad1, _quad2;
+
+
+ private final String _addDecodedToSymbols(int len, String name)
+ {
+ if (len < 5) {
+ return _symbols.addName(name, _quad1, 0).getName();
+ }
+ if (len < 9) {
+ return _symbols.addName(name, _quad1, _quad2).getName();
+ }
+ int qlen = (len + 3) >> 2;
+ return _symbols.addName(name, _quadBuffer, qlen).getName();
+ }
+
+ private final String _decodeShortAsciiName(int len)
+ throws IOException, JsonParseException
+ {
+ // note: caller ensures we have enough bytes available
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int outPtr = 0;
+ final byte[] inBuf = _inputBuffer;
+ int inPtr = _inputPtr;
+
+ // loop unrolling seems to help here:
+ for (int inEnd = inPtr + len - 3; inPtr < inEnd; ) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ int left = (len & 3);
+ if (left > 0) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 1) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ if (left > 2) {
+ outBuf[outPtr++] = (char) inBuf[inPtr++];
+ }
+ }
+ }
+ _inputPtr = inPtr;
+ _textBuffer.setCurrentLength(len);
+ return _textBuffer.contentsAsString();
+ }
+
+ /**
+ * Helper method used to decode short Unicode string, length for which actual
+ * length (in bytes) is known
+ *
+ * @param len Length between 1 and 64
+ */
+ private final String _decodeShortUnicodeName(int len)
+ throws IOException, JsonParseException
+ {
+ // note: caller ensures we have enough bytes available
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ int inPtr = _inputPtr;
+ _inputPtr += len;
+ final int[] codes = SmileConstants.sUtf8UnitLengths;
+ final byte[] inBuf = _inputBuffer;
+ for (int end = inPtr + len; inPtr < end; ) {
+ int i = inBuf[inPtr++] & 0xFF;
+ int code = codes[i];
+ if (code != 0) {
+ // trickiest one, need surrogate handling
+ switch (code) {
+ case 1:
+ i = ((i & 0x1F) << 6) | (inBuf[inPtr++] & 0x3F);
+ break;
+ case 2:
+ i = ((i & 0x0F) << 12)
+ | ((inBuf[inPtr++] & 0x3F) << 6)
+ | (inBuf[inPtr++] & 0x3F);
+ break;
+ case 3:
+ i = ((i & 0x07) << 18)
+ | ((inBuf[inPtr++] & 0x3F) << 12)
+ | ((inBuf[inPtr++] & 0x3F) << 6)
+ | (inBuf[inPtr++] & 0x3F);
+ // note: this is the codepoint value; need to split, too
+ i -= 0x10000;
+ outBuf[outPtr++] = (char) (0xD800 | (i >> 10));
+ i = 0xDC00 | (i & 0x3FF);
+ break;
+ default: // invalid
+ _reportError("Invalid byte "+Integer.toHexString(i)+" in short Unicode text block");
+ }
+ }
+ outBuf[outPtr++] = (char) i;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ return _textBuffer.contentsAsString();
+ }
+
+ // note: slightly edited copy of UTF8StreamParser.addName()
+ private final Name _decodeLongUnicodeName(int[] quads, int byteLen, int quadLen)
+ throws IOException, JsonParseException
+ {
+ int lastQuadBytes = byteLen & 3;
+ // Ok: must decode UTF-8 chars. No other validation SHOULD be needed (except bounds checks?)
+ /* Note: last quad is not correctly aligned (leading zero bytes instead
+ * need to shift a bit, instead of trailing). Only need to shift it
+ * for UTF-8 decoding; need revert for storage (since key will not
+ * be aligned, to optimize lookup speed)
+ */
+ int lastQuad;
+
+ if (lastQuadBytes < 4) {
+ lastQuad = quads[quadLen-1];
+ // 8/16/24 bit left shift
+ quads[quadLen-1] = (lastQuad << ((4 - lastQuadBytes) << 3));
+ } else {
+ lastQuad = 0;
+ }
+
+ char[] cbuf = _textBuffer.emptyAndGetCurrentSegment();
+ int cix = 0;
+
+ for (int ix = 0; ix < byteLen; ) {
+ int ch = quads[ix >> 2]; // current quad, need to shift+mask
+ int byteIx = (ix & 3);
+ ch = (ch >> ((3 - byteIx) << 3)) & 0xFF;
+ ++ix;
+
+ if (ch > 127) { // multi-byte
+ int needed;
+ if ((ch & 0xE0) == 0xC0) { // 2 bytes (0x0080 - 0x07FF)
+ ch &= 0x1F;
+ needed = 1;
+ } else if ((ch & 0xF0) == 0xE0) { // 3 bytes (0x0800 - 0xFFFF)
+ ch &= 0x0F;
+ needed = 2;
+ } else if ((ch & 0xF8) == 0xF0) { // 4 bytes; double-char with surrogates and all...
+ ch &= 0x07;
+ needed = 3;
+ } else { // 5- and 6-byte chars not valid chars
+ _reportInvalidInitial(ch);
+ needed = ch = 1; // never really gets this far
+ }
+ if ((ix + needed) > byteLen) {
+ _reportInvalidEOF(" in long field name");
+ }
+
+ // Ok, always need at least one more:
+ int ch2 = quads[ix >> 2]; // current quad, need to shift+mask
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ if (needed > 1) {
+ ch2 = quads[ix >> 2];
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ if (needed > 2) { // 4 bytes? (need surrogates on output)
+ ch2 = quads[ix >> 2];
+ byteIx = (ix & 3);
+ ch2 = (ch2 >> ((3 - byteIx) << 3));
+ ++ix;
+ if ((ch2 & 0xC0) != 0x080) {
+ _reportInvalidOther(ch2 & 0xFF);
+ }
+ ch = (ch << 6) | (ch2 & 0x3F);
+ }
+ }
+ if (needed > 2) { // surrogate pair? once again, let's output one here, one later on
+ ch -= 0x10000; // to normalize it starting with 0x0
+ if (cix >= cbuf.length) {
+ cbuf = _textBuffer.expandCurrentSegment();
+ }
+ cbuf[cix++] = (char) (0xD800 + (ch >> 10));
+ ch = 0xDC00 | (ch & 0x03FF);
+ }
+ }
+ if (cix >= cbuf.length) {
+ cbuf = _textBuffer.expandCurrentSegment();
+ }
+ cbuf[cix++] = (char) ch;
+ }
+
+ // Ok. Now we have the character array, and can construct the String
+ String baseName = new String(cbuf, 0, cix);
+ // And finally, un-align if necessary
+ if (lastQuadBytes < 4) {
+ quads[quadLen-1] = lastQuad;
+ }
+ return _symbols.addName(baseName, quads, quadLen);
+ }
+
+ private final void _handleLongFieldName() throws IOException, JsonParseException
+ {
+ // First: gather quads we need, looking for end marker
+ final byte[] inBuf = _inputBuffer;
+ int quads = 0;
+ int bytes = 0;
+ int q = 0;
+
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ byte b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 0;
+ break;
+ }
+ q = ((int) b) & 0xFF;
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 1;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 2;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ b = inBuf[_inputPtr++];
+ if (BYTE_MARKER_END_OF_STRING == b) {
+ bytes = 3;
+ break;
+ }
+ q = (q << 8) | (b & 0xFF);
+ if (quads >= _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, _quadBuffer.length + 256); // grow by 1k
+ }
+ _quadBuffer[quads++] = q;
+ }
+ // and if we have more bytes, append those too
+ int byteLen = (quads << 2);
+ if (bytes > 0) {
+ if (quads >= _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, _quadBuffer.length + 256);
+ }
+ _quadBuffer[quads++] = q;
+ byteLen += bytes;
+ }
+
+ // Know this name already?
+ String name;
+ Name n = _symbols.findName(_quadBuffer, quads);
+ if (n != null) {
+ name = n.getName();
+ } else {
+ name = _decodeLongUnicodeName(_quadBuffer, byteLen, quads).getName();
+ }
+ if (_seenNames != null) {
+ if (_seenNameCount >= _seenNames.length) {
+ _seenNames = _expandSeenNames(_seenNames);
+ }
+ _seenNames[_seenNameCount++] = name;
+ }
+ _parsingContext.setCurrentName(name);
+ }
+
+ /**
+ * Helper method for trying to find specified encoded UTF-8 byte sequence
+ * from symbol table; if successful avoids actual decoding to String
+ */
+ private final Name _findDecodedFromSymbols(int len)
+ throws IOException, JsonParseException
+ {
+ if ((_inputEnd - _inputPtr) < len) {
+ _loadToHaveAtLeast(len);
+ }
+ // First: maybe we already have this name decoded?
+ if (len < 5) {
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ int q = inBuf[inPtr] & 0xFF;
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ }
+ _quad1 = q;
+ return _symbols.findName(q);
+ }
+ if (len < 9) {
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ // First quadbyte is easy
+ int q1 = (inBuf[inPtr] & 0xFF) << 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ q1 <<= 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ q1 <<= 8;
+ q1 += (inBuf[++inPtr] & 0xFF);
+ int q2 = (inBuf[++inPtr] & 0xFF);
+ len -= 5;
+ if (len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q2 = (q2 << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ }
+ _quad1 = q1;
+ _quad2 = q2;
+ return _symbols.findName(q1, q2);
+ }
+ return _findDecodedMedium(len);
+ }
+
+ /**
+ * Method for locating names longer than 8 bytes (in UTF-8)
+ */
+ private final Name _findDecodedMedium(int len)
+ throws IOException, JsonParseException
+ {
+ // first, need enough buffer to store bytes as ints:
+ {
+ int bufLen = (len + 3) >> 2;
+ if (bufLen > _quadBuffer.length) {
+ _quadBuffer = _growArrayTo(_quadBuffer, bufLen);
+ }
+ }
+ // then decode, full quads first
+ int offset = 0;
+ int inPtr = _inputPtr;
+ final byte[] inBuf = _inputBuffer;
+ do {
+ int q = (inBuf[inPtr++] & 0xFF) << 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ q <<= 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ q <<= 8;
+ q |= inBuf[inPtr++] & 0xFF;
+ _quadBuffer[offset++] = q;
+ } while ((len -= 4) > 3);
+ // and then leftovers
+ if (len > 0) {
+ int q = inBuf[inPtr] & 0xFF;
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ if (--len > 0) {
+ q = (q << 8) + (inBuf[++inPtr] & 0xFF);
+ }
+ }
+ _quadBuffer[offset++] = q;
+ }
+ return _symbols.findName(_quadBuffer, offset);
+ }
+
+ private static int[] _growArrayTo(int[] arr, int minSize)
+ {
+ int[] newArray = new int[minSize + 4];
+ if (arr != null) {
+ // !!! TODO: JDK 1.6, Arrays.copyOf
+ System.arraycopy(arr, 0, newArray, 0, arr.length);
+ }
+ return newArray;
+ }
+
+ /*
+ /**********************************************************************
+ /* Internal methods, secondary parsing
+ /**********************************************************************
+ */
+
+ @Override
+ protected void _parseNumericValue(int expType)
+ throws IOException, JsonParseException
+ {
+ if (_tokenIncomplete) {
+ _reportError("No current token available, can not call accessors");
+ }
+ }
+
+ /**
+ * Method called to finish parsing of a token, given partial decoded
+ * state.
+ */
+ protected final JsonToken _finishToken()
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ byte b = _inputBuffer[_inputPtr++];
+
+ // first need to handle possible header, since that is usually not
+ // exposed as an event (expect when it implies document boundary)
+ if (_state == STATE_INITIAL) { // just need to see if we see something like header:
+ if (b == SmileConstants.HEADER_BYTE_1) {
+ if (!_handleHeader(0)) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ // if handled, get next byte to code (if available)
+ if (_inputPtr >= _inputEnd) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ b = _inputBuffer[_inputPtr++];
+ } else {
+ // nope, not header marker.
+ // header mandatory? not good...
+ if (_cfgRequireHeader) {
+ String msg;
+ if (b == '{' || b == '[') {
+ msg = "Input does not start with Smile format header (first byte = 0x"
+ +Integer.toHexString(b & 0xFF)+") -- rather, it starts with '"+((char) b)
+ +"' (plain JSON input?) -- can not parse";
+ } else {
+ msg = "Input does not start with Smile format header (first byte = 0x"
+ +Integer.toHexString(b & 0xFF)+") and parser has REQUIRE_HEADER enabled: can not parse";
+ }
+ throw new JsonParseException(msg, JsonLocation.NA);
+ }
+ }
+ // otherwise, fall through, with byte (_handleHeader has set _state)
+ } else if (_state == STATE_HEADER) { // in-stream header
+ if (!_handleHeader(_substate)) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ // is it enough to leave '_tokenIncomplete' false here?
+ if (_inputPtr >= _inputEnd) {
+ return JsonToken.NOT_AVAILABLE;
+ }
+ b = _inputBuffer[_inputPtr++];
+ // fall through
+ }
+
+ switch (_state) {
+ case STATE_NUMBER_INT:
+ return _nextInt(_substate, _pendingInt);
+ case STATE_NUMBER_LONG:
+ return _nextLong(_substate, _pendingLong);
+ case STATE_NUMBER_BIGINT:
+ return _nextBigInt(_substate);
+ case STATE_NUMBER_FLOAT:
+ return _nextFloat(_substate, _pendingInt) ;
+ case STATE_NUMBER_DOUBLE:
+ return _nextDouble(_substate, _pendingLong);
+ case STATE_NUMBER_BIGDEC:
+ return _nextBigDecimal(_substate);
+ }
+ _throwInvalidState("Illegal state when trying to complete token: ");
+ return null;
+ }
+
+ /*
+ /**********************************************************************
+ /* Internal methods, UTF8 decoding
+ /**********************************************************************
+ */
+
+ /*
+ private final int _decodeUtf8_2(int c)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ return ((c & 0x1F) << 6) | (d & 0x3F);
+ }
+
+ private final int _decodeUtf8_3(int c1)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ c1 &= 0x0F;
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ int c = (c1 << 6) | (d & 0x3F);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ return c;
+ }
+
+ private final int _decodeUtf8_3fast(int c1)
+ throws IOException, JsonParseException
+ {
+ c1 &= 0x0F;
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ int c = (c1 << 6) | (d & 0x3F);
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ return c;
+ }
+
+ // @return Character value <b>minus 0x10000</c>; this so that caller
+ // can readily expand it to actual surrogates
+ private final int _decodeUtf8_4(int c)
+ throws IOException, JsonParseException
+ {
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ int d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = ((c & 0x07) << 6) | (d & 0x3F);
+
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+ c = (c << 6) | (d & 0x3F);
+ if (_inputPtr >= _inputEnd) {
+ loadMoreGuaranteed();
+ }
+ d = (int) _inputBuffer[_inputPtr++];
+ if ((d & 0xC0) != 0x080) {
+ _reportInvalidOther(d & 0xFF, _inputPtr);
+ }
+
+ // note: won't change it to negative here, since caller
+ // already knows it'll need a surrogate
+ return ((c << 6) | (d & 0x3F)) - 0x10000;
+ }
+*/
+
+ /*
+ /**********************************************************************
+ /* Internal methods, error reporting
+ /**********************************************************************
+ */
+
+ protected void _reportInvalidSharedName(int index) throws IOException
+ {
+ if (_seenNames == null) {
+ _reportError("Encountered shared name reference, even though document header explicitly declared no shared name references are included");
+ }
+ _reportError("Invalid shared name reference "+index+"; only got "+_seenNameCount+" names in buffer (invalid content)");
+ }
+
+ protected void _reportInvalidSharedStringValue(int index) throws IOException
+ {
+ if (_seenStringValues == null) {
+ _reportError("Encountered shared text value reference, even though document header did not declared shared text value references may be included");
+ }
+ _reportError("Invalid shared text value reference "+index+"; only got "+_seenStringValueCount+" names in buffer (invalid content)");
+ }
+
+ protected void _reportInvalidChar(int c) throws JsonParseException
+ {
+ // Either invalid WS or illegal UTF-8 start char
+ if (c < ' ') {
+ _throwInvalidSpace(c);
+ }
+ _reportInvalidInitial(c);
+ }
+
+ protected void _reportInvalidInitial(int mask)
+ throws JsonParseException
+ {
+ _reportError("Invalid UTF-8 start byte 0x"+Integer.toHexString(mask));
+ }
+
+ protected void _reportInvalidOther(int mask)
+ throws JsonParseException
+ {
+ _reportError("Invalid UTF-8 middle byte 0x"+Integer.toHexString(mask));
+ }
+
+ protected void _reportInvalidOther(int mask, int ptr)
+ throws JsonParseException
+ {
+ _inputPtr = ptr;
+ _reportInvalidOther(mask);
+ }
+
+ protected void _throwInvalidState(String desc)
+ {
+ throw new IllegalStateException(desc+": state="+_state);
+ }
+}
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/async/package-info.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/package-info.java
new file mode 100644
index 0000000..14e891d
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/async/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * Package that contains experimental non-blocking ("asynchronous")
+ * implementation of reader-part of Jackson streaming API,
+ * working on Smile format.
+ */
+package com.fasterxml.jackson.dataformat.smile.async;
diff --git a/src/main/java/com/fasterxml/jackson/dataformat/smile/package-info.java b/src/main/java/com/fasterxml/jackson/dataformat/smile/package-info.java
new file mode 100644
index 0000000..b14fdaa
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/dataformat/smile/package-info.java
@@ -0,0 +1,7 @@
+/**
+ * Package that contains implementation of Jackson Streaming API that works
+ * with Smile data format (see <a href="http://wiki.fasterxml.com/SmileFormat">Smile format specification</a>),
+ * and can be used with standard Jackson data binding functionality to deal with
+ * Smile encoded data.
+ */
+package com.fasterxml.jackson.dataformat.smile;
diff --git a/src/main/resources/META-INF/services/com.fasterxml.jackson.core.JsonFactory b/src/main/resources/META-INF/services/com.fasterxml.jackson.core.JsonFactory
new file mode 100644
index 0000000..9ad3031
--- /dev/null
+++ b/src/main/resources/META-INF/services/com.fasterxml.jackson.core.JsonFactory
@@ -0,0 +1,15 @@
+# Copyright 2012 FasterXML.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+com.fasterxml.jackson.dataformat.smile.SmileFactory
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/SmileTestBase.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/SmileTestBase.java
new file mode 100644
index 0000000..aa40b33
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/SmileTestBase.java
@@ -0,0 +1,223 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.junit.Assert;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+abstract class SmileTestBase
+ extends junit.framework.TestCase
+{
+ // From JSON specification, sample doc...
+ protected final static int SAMPLE_SPEC_VALUE_WIDTH = 800;
+ protected final static int SAMPLE_SPEC_VALUE_HEIGHT = 600;
+ protected final static String SAMPLE_SPEC_VALUE_TITLE = "View from 15th Floor";
+ protected final static String SAMPLE_SPEC_VALUE_TN_URL = "http://www.example.com/image/481989943";
+ protected final static int SAMPLE_SPEC_VALUE_TN_HEIGHT = 125;
+ protected final static String SAMPLE_SPEC_VALUE_TN_WIDTH = "100";
+ protected final static int SAMPLE_SPEC_VALUE_TN_ID1 = 116;
+ protected final static int SAMPLE_SPEC_VALUE_TN_ID2 = 943;
+ protected final static int SAMPLE_SPEC_VALUE_TN_ID3 = 234;
+ protected final static int SAMPLE_SPEC_VALUE_TN_ID4 = 38793;
+
+ protected final static String SAMPLE_DOC_JSON_SPEC =
+ "{\n"
+ +" \"Image\" : {\n"
+ +" \"Width\" : "+SAMPLE_SPEC_VALUE_WIDTH+",\n"
+ +" \"Height\" : "+SAMPLE_SPEC_VALUE_HEIGHT+","
+ +"\"Title\" : \""+SAMPLE_SPEC_VALUE_TITLE+"\",\n"
+ +" \"Thumbnail\" : {\n"
+ +" \"Url\" : \""+SAMPLE_SPEC_VALUE_TN_URL+"\",\n"
+ +"\"Height\" : "+SAMPLE_SPEC_VALUE_TN_HEIGHT+",\n"
+ +" \"Width\" : \""+SAMPLE_SPEC_VALUE_TN_WIDTH+"\"\n"
+ +" },\n"
+ +" \"IDs\" : ["+SAMPLE_SPEC_VALUE_TN_ID1+","+SAMPLE_SPEC_VALUE_TN_ID2+","+SAMPLE_SPEC_VALUE_TN_ID3+","+SAMPLE_SPEC_VALUE_TN_ID4+"]\n"
+ +" }"
+ +"}"
+ ;
+
+ /*
+ /**********************************************************
+ /* Factory methods
+ /**********************************************************
+ */
+
+ protected SmileParser _smileParser(byte[] input) throws IOException {
+ return _smileParser(input, false);
+ }
+
+ protected SmileParser _smileParser(InputStream in) throws IOException {
+ return _smileParser(in, false);
+ }
+
+ protected SmileParser _smileParser(byte[] input, boolean requireHeader) throws IOException
+ {
+ SmileFactory f = smileFactory(requireHeader, false, false);
+ return _smileParser(f, input);
+ }
+
+ protected SmileParser _smileParser(InputStream in, boolean requireHeader) throws IOException
+ {
+ SmileFactory f = smileFactory(requireHeader, false, false);
+ return _smileParser(f, in);
+ }
+
+ protected SmileParser _smileParser(SmileFactory f, byte[] input) throws IOException {
+ return f.createParser(input);
+ }
+
+ protected SmileParser _smileParser(SmileFactory f, InputStream in) throws IOException {
+ return f.createParser(in);
+ }
+
+ protected ObjectMapper smileMapper() {
+ return smileMapper(false);
+ }
+
+ protected ObjectMapper smileMapper(boolean requireHeader) {
+ return smileMapper(requireHeader, false, false);
+ }
+
+ protected ObjectMapper smileMapper(boolean requireHeader,
+ boolean writeHeader, boolean writeEndMarker)
+ {
+ return new ObjectMapper(smileFactory(requireHeader, writeHeader, writeEndMarker));
+ }
+
+ protected SmileFactory smileFactory(boolean requireHeader,
+ boolean writeHeader, boolean writeEndMarker)
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileParser.Feature.REQUIRE_HEADER, requireHeader);
+ f.configure(SmileGenerator.Feature.WRITE_HEADER, writeHeader);
+ f.configure(SmileGenerator.Feature.WRITE_END_MARKER, writeEndMarker);
+ return f;
+ }
+
+ protected byte[] _smileDoc(String json) throws IOException
+ {
+ return _smileDoc(json, true);
+ }
+
+ protected byte[] _smileDoc(String json, boolean writeHeader) throws IOException
+ {
+ return _smileDoc(new SmileFactory(), json, writeHeader);
+ }
+
+ protected byte[] _smileDoc(SmileFactory smileFactory, String json, boolean writeHeader) throws IOException
+ {
+ JsonFactory jf = new JsonFactory();
+ JsonParser jp = jf.createParser(json);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ JsonGenerator jg = smileGenerator(out, writeHeader);
+
+ while (jp.nextToken() != null) {
+ jg.copyCurrentEvent(jp);
+ }
+ jp.close();
+ jg.close();
+ return out.toByteArray();
+ }
+
+ protected SmileGenerator smileGenerator(ByteArrayOutputStream result, boolean addHeader)
+ throws IOException
+ {
+ return smileGenerator(new SmileFactory(), result, addHeader);
+ }
+
+ protected SmileGenerator smileGenerator(SmileFactory f,
+ ByteArrayOutputStream result, boolean addHeader)
+ throws IOException
+ {
+ f.configure(SmileGenerator.Feature.WRITE_HEADER, addHeader);
+ return f.createGenerator(result, null);
+ }
+
+ /*
+ /**********************************************************
+ /* Additional assertion methods
+ /**********************************************************
+ */
+
+ protected void assertToken(JsonToken expToken, JsonToken actToken)
+ {
+ if (actToken != expToken) {
+ fail("Expected token "+expToken+", current token "+actToken);
+ }
+ }
+
+ protected void assertToken(JsonToken expToken, JsonParser jp)
+ {
+ assertToken(expToken, jp.getCurrentToken());
+ }
+
+ protected void assertType(Object ob, Class<?> expType)
+ {
+ if (ob == null) {
+ fail("Expected an object of type "+expType.getName()+", got null");
+ }
+ Class<?> cls = ob.getClass();
+ if (!expType.isAssignableFrom(cls)) {
+ fail("Expected type "+expType.getName()+", got "+cls.getName());
+ }
+ }
+
+ protected void verifyException(Throwable e, String... matches)
+ {
+ String msg = e.getMessage();
+ String lmsg = (msg == null) ? "" : msg.toLowerCase();
+ for (String match : matches) {
+ String lmatch = match.toLowerCase();
+ if (lmsg.indexOf(lmatch) >= 0) {
+ return;
+ }
+ }
+ fail("Expected an exception with one of substrings ("+Arrays.asList(matches)+"): got one with message \""+msg+"\"");
+ }
+
+ protected void _verifyBytes(byte[] actBytes, byte... expBytes)
+ {
+ Assert.assertArrayEquals(expBytes, actBytes);
+ }
+
+ /**
+ * Method that gets textual contents of the current token using
+ * available methods, and ensures results are consistent, before
+ * returning them
+ */
+ protected String getAndVerifyText(JsonParser jp)
+ throws IOException, JsonParseException
+ {
+ // Ok, let's verify other accessors
+ int actLen = jp.getTextLength();
+ char[] ch = jp.getTextCharacters();
+ String str2 = new String(ch, jp.getTextOffset(), actLen);
+ String str = jp.getText();
+
+ if (str.length() != actLen) {
+ fail("Internal problem (jp.token == "+jp.getCurrentToken()+"): jp.getText().length() ['"+str+"'] == "+str.length()+"; jp.getTextLength() == "+actLen);
+ }
+ assertEquals("String access via getText(), getTextXxx() must be the same", str, str2);
+
+ return str;
+ }
+
+ /*
+ /**********************************************************
+ /* Other helper methods
+ /**********************************************************
+ */
+
+ public String quote(String str) {
+ return '"'+str+'"';
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestDocBoundary.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestDocBoundary.java
new file mode 100644
index 0000000..37ac27f
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestDocBoundary.java
@@ -0,0 +1,109 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+/**
+ * Unit tests for verifying that multiple document output and document
+ * boundaries and/or header mark handling works as expected
+ */
+public class TestDocBoundary
+ extends SmileTestBase
+{
+ public void testNoHeadersNoEndMarker() throws Exception
+ {
+ _verifyMultiDoc(false, false);
+ }
+
+ public void testHeadersNoEndMarker() throws Exception
+ {
+ _verifyMultiDoc(true, false);
+ }
+
+ public void testEndMarkerNoHeader() throws Exception
+ {
+ _verifyMultiDoc(false, true);
+ }
+
+ public void testHeaderAndEndMarker() throws Exception
+ {
+ _verifyMultiDoc(true, true);
+ }
+
+ public void testExtraHeader() throws Exception
+ {
+ // also; sprinkling headers can be used to segment document
+ for (boolean addHeader : new boolean[] { false, true }) {
+ SmileFactory f = smileFactory(false, false, false);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator jg = f.createGenerator(out);
+ jg.writeNumber(1);
+ if (addHeader) jg.writeHeader();
+ jg.writeNumber(2);
+ if (addHeader) jg.writeHeader();
+ jg.writeNumber(3);
+ jg.close();
+
+ SmileParser jp = f.createParser(out.toByteArray());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(1, jp.getIntValue());
+ if (addHeader) {
+ assertNull(jp.nextToken());
+ }
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(2, jp.getIntValue());
+ if (addHeader) {
+ assertNull(jp.nextToken());
+ }
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(3, jp.getIntValue());
+ assertNull(jp.nextToken());
+ jg.close();
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ protected void _verifyMultiDoc(boolean addHeader, boolean addEndMarker) throws Exception
+ {
+ SmileFactory f = smileFactory(false, addHeader, addEndMarker);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator jg = f.createGenerator(out);
+ // First doc, JSON Object
+ jg.writeStartObject();
+ jg.writeEndObject();
+ jg.close();
+ // and second, array
+ jg = f.createGenerator(out);
+ jg.writeStartArray();
+ jg.writeEndArray();
+ jg.close();
+
+ // and read it back
+ SmileParser jp = f.createParser(out.toByteArray());
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+
+ // now: if one of header or end marker (or, both) enabled, should get null here:
+ if (addHeader || addEndMarker) {
+ assertNull(jp.nextToken());
+ }
+
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+
+ // end
+ assertNull(jp.nextToken());
+ // and no more:
+ assertNull(jp.nextToken());
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFeatures.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFeatures.java
new file mode 100644
index 0000000..6a68f0d
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFeatures.java
@@ -0,0 +1,37 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import com.fasterxml.jackson.databind.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+public class TestFeatures
+ extends SmileTestBase
+{
+ /*
+ /**********************************************************
+ /* Helper types
+ /**********************************************************
+ */
+
+ static class Bean {
+ public int value;
+ }
+
+ /*
+ /**********************************************************
+ /* Unit tests
+ /**********************************************************
+ */
+ // Let's ensure indentation doesn't break anything (should be NOP)
+ public void testIndent() throws Exception
+ {
+ ObjectMapper mapper = new ObjectMapper(new SmileFactory());
+ mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
+ Bean bean = new Bean();
+ bean.value = 42;
+
+ byte[] smile = mapper.writeValueAsBytes(bean);
+ Bean result = mapper.readValue(smile, 0, smile.length, Bean.class);
+ assertEquals(42, result.value);
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFormatDetection.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFormatDetection.java
new file mode 100644
index 0000000..cf54b55
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestFormatDetection.java
@@ -0,0 +1,44 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import com.fasterxml.jackson.databind.*;
+
+public class TestFormatDetection extends SmileTestBase
+{
+ static class POJO {
+ public int id;
+ public String name;
+
+ public POJO() { }
+ public POJO(int id, String name)
+ {
+ this.id = id;
+ this.name = name;
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Test methods
+ /**********************************************************
+ */
+
+ public void testSimple() throws Exception
+ {
+ final ObjectMapper mapper = new ObjectMapper();
+ final ObjectReader jsonReader = mapper.reader(POJO.class);
+ final String JSON = "{\"name\":\"Bob\", \"id\":3}";
+
+ byte[] doc = _smileDoc(JSON, true);
+
+ ObjectReader detecting = jsonReader.withFormatDetection(jsonReader,
+ jsonReader.with(new SmileFactory()));
+ POJO pojo = detecting.readValue(doc);
+ assertEquals(3, pojo.id);
+ assertEquals("Bob", pojo.name);
+
+ // let's verify it also works for plain JSON...
+ pojo = detecting.readValue(JSON.getBytes("UTF-8"));
+ assertEquals(3, pojo.id);
+ assertEquals("Bob", pojo.name);
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGenerator.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGenerator.java
new file mode 100644
index 0000000..f265afb
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGenerator.java
@@ -0,0 +1,246 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.util.HashMap;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileConstants;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileUtil;
+
+import static com.fasterxml.jackson.dataformat.smile.SmileConstants.*;
+
+public class TestGenerator
+ extends SmileTestBase
+{
+ /**
+ * Test for verifying handling of 'true', 'false' and 'null' literals
+ */
+ public void testSimpleLiterals() throws Exception
+ {
+ // false, no header (or frame marker)
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeBoolean(true);
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_TRUE);
+
+ // false, no header or frame marker
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeBoolean(false);
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_FALSE);
+
+ // null, no header or frame marker
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNull();
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_NULL);
+
+ // And then with some other combinations:
+ // true, but with header
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, true);
+ gen.writeBoolean(true);
+ gen.close();
+
+ // note: version, and 'check shared names', but not 'check shared strings' or 'raw binary'
+ int b4 = HEADER_BYTE_4 | SmileConstants.HEADER_BIT_HAS_SHARED_NAMES;
+
+ _verifyBytes(out.toByteArray(),
+ HEADER_BYTE_1, HEADER_BYTE_2, HEADER_BYTE_3, (byte) b4,
+ SmileConstants.TOKEN_LITERAL_TRUE);
+
+ // null, with header and end marker
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, true);
+ gen.enable(SmileGenerator.Feature.WRITE_END_MARKER);
+ gen.writeNull();
+ gen.close();
+ _verifyBytes(out.toByteArray(),
+ HEADER_BYTE_1, HEADER_BYTE_2, HEADER_BYTE_3, (byte) b4,
+ TOKEN_LITERAL_NULL, BYTE_MARKER_END_OF_CONTENT);
+ }
+
+ public void testSimpleArray() throws Exception
+ {
+ // First: empty array (2 bytes)
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartArray();
+ gen.writeEndArray();
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_START_ARRAY,
+ SmileConstants.TOKEN_LITERAL_END_ARRAY);
+
+ // then simple array with 3 literals
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeStartArray();
+ gen.writeBoolean(true);
+ gen.writeNull();
+ gen.writeBoolean(false);
+ gen.writeEndArray();
+ gen.close();
+ assertEquals(5, out.toByteArray().length);
+
+ // and then array containing another array and short String
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeStartArray();
+ gen.writeStartArray();
+ gen.writeEndArray();
+ gen.writeString("12");
+ gen.writeEndArray();
+ gen.close();
+ // 4 bytes for start/end arrays; 3 bytes for short ascii string
+ assertEquals(7, out.toByteArray().length);
+ }
+
+ public void testShortAscii() throws Exception
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeString("abc");
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte)0x42, (byte) 'a', (byte) 'b', (byte) 'c');
+ }
+
+
+ public void testTrivialObject() throws Exception
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartObject();
+ gen.writeNumberField("a", 6);
+ gen.writeEndObject();
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_START_OBJECT,
+ (byte) 0x80, (byte) 'a', (byte) (0xC0 + SmileUtil.zigzagEncode(6)),
+ SmileConstants.TOKEN_LITERAL_END_OBJECT);
+ }
+
+ public void test2FieldObject() throws Exception
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartObject();
+ gen.writeNumberField("a", 1);
+ gen.writeNumberField("b", 2);
+ gen.writeEndObject();
+ gen.close();
+ _verifyBytes(out.toByteArray(), SmileConstants.TOKEN_LITERAL_START_OBJECT,
+ (byte) 0x80, (byte) 'a', (byte) (0xC0 + SmileUtil.zigzagEncode(1)),
+ (byte) 0x80, (byte) 'b', (byte) (0xC0 + SmileUtil.zigzagEncode(2)),
+ SmileConstants.TOKEN_LITERAL_END_OBJECT);
+ }
+
+ public void testAnotherObject() throws Exception
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartObject();
+ gen.writeNumberField("a", 8);
+ gen.writeFieldName("b");
+ gen.writeStartArray();
+ gen.writeBoolean(true);
+ gen.writeEndArray();
+ gen.writeFieldName("c");
+ gen.writeStartObject();
+ gen.writeEndObject();
+
+ gen.writeFieldName("d");
+ gen.writeStartObject();
+ gen.writeFieldName("3");
+ gen.writeNull();
+ gen.writeEndObject();
+
+ gen.writeEndObject();
+ gen.close();
+ assertEquals(21, out.toByteArray().length);
+ }
+
+ /**
+ * Test to verify that
+ */
+ public void testSharedStrings() throws Exception
+ {
+ // first, no sharing, 2 separate Strings
+ final String VALUE = "abcde12345";
+ byte[] data = writeRepeatedString(false, VALUE);
+ int BASE_LEN = 28;
+ assertEquals(BASE_LEN, data.length);
+ data = writeRepeatedString(true, VALUE);
+ if (data.length >= BASE_LEN) { // should be less
+ fail("Expected shared String length to be < "+BASE_LEN+", was "+data.length);
+ }
+ }
+
+ public void testWithMap() throws Exception
+ {
+ final SmileFactory smileFactory = new SmileFactory();
+ smileFactory.disable(SmileGenerator.Feature.WRITE_HEADER);
+ smileFactory.disable(SmileParser.Feature.REQUIRE_HEADER);
+ final ObjectMapper smileObjectMapper = new ObjectMapper(smileFactory);
+ final HashMap<String, String> data = new HashMap<String,String>();
+ data.put("key", "value");
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final SmileGenerator smileGenerator = smileFactory.createGenerator(out);
+ // NOTE: not optimal way -- should use "gen.writeStartArray()" -- but exposed a problem
+ out.write(SmileConstants.TOKEN_LITERAL_START_ARRAY);
+ smileObjectMapper.writeValue(smileGenerator, data);
+ smileGenerator.flush();
+ // as above, should use generator
+ out.write(SmileConstants.TOKEN_LITERAL_END_ARRAY);
+ smileGenerator.close();
+ byte[] doc = out.toByteArray();
+ JsonNode root = smileObjectMapper.readTree(doc);
+ assertNotNull(root);
+ assertTrue(root.isArray());
+ assertEquals(1, root.size());
+ }
+
+ // [Issue#6], missing overrides for File-backed generator
+ public void testWriteToFile() throws Exception
+ {
+ final SmileFactory smileFactory = new SmileFactory();
+ ObjectMapper mapper = new ObjectMapper(smileFactory);
+ File f = File.createTempFile("test", ".tst");
+ mapper.writeValue(f, Integer.valueOf(3));
+
+ JsonParser jp = smileFactory.createParser(f);
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(3, jp.getIntValue());
+ assertNull(jp.nextToken());
+ jp.close();
+ f.delete();
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private byte[] writeRepeatedString(boolean shared, String value) throws Exception
+ {
+ SmileFactory f = new SmileFactory();
+ // need header to enable shared string values
+ f.configure(SmileGenerator.Feature.WRITE_HEADER, true);
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, shared);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = f.createGenerator(out);
+ gen.writeStartArray();
+ gen.writeString(value);
+ gen.writeString(value);
+ gen.writeEndArray();
+ gen.close();
+ return out.toByteArray();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBinary.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBinary.java
new file mode 100644
index 0000000..49a870d
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBinary.java
@@ -0,0 +1,97 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import org.junit.Assert;
+
+import com.fasterxml.jackson.core.*;
+
+public class TestGeneratorBinary extends SmileTestBase
+{
+ static class ThrottledInputStream extends FilterInputStream
+ {
+ protected final int _maxBytes;
+
+ public ThrottledInputStream(byte[] data, int maxBytes)
+ {
+ this(new ByteArrayInputStream(data), maxBytes);
+ }
+
+ public ThrottledInputStream(InputStream in, int maxBytes)
+ {
+ super(in);
+ _maxBytes = maxBytes;
+ }
+
+ @Override
+ public int read(byte[] buf) throws IOException {
+ return read(buf, 0, buf.length);
+ }
+
+ @Override
+ public int read(byte[] buf, int offset, int len) throws IOException {
+ return in.read(buf, offset, Math.min(_maxBytes, len));
+ }
+
+ }
+
+ /*
+ /**********************************************************
+ /* Test methods
+ /**********************************************************
+ */
+
+ public void testStreamingBinary() throws Exception
+ {
+ _testStreamingBinary(true);
+ _testStreamingBinary(false);
+ }
+
+ public void testBinaryWithoutLength() throws Exception
+ {
+ final SmileFactory f = new SmileFactory();
+ JsonGenerator jg = f.createGenerator(new ByteArrayOutputStream());
+ try {
+ jg.writeBinary(new ByteArrayInputStream(new byte[1]), -1);
+ fail("Should have failed");
+ } catch (UnsupportedOperationException e) {
+ verifyException(e, "must pass actual length");
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private final static String TEXT = "Some content so that we can test encoding of base64 data; must"
+ +" be long enough include a line wrap or two...";
+ private final static String TEXT4 = TEXT + TEXT + TEXT + TEXT;
+
+ private void _testStreamingBinary(boolean rawBinary) throws Exception
+ {
+ final SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !rawBinary);
+
+ final byte[] INPUT = TEXT4.getBytes("UTF-8");
+ for (int chunkSize : new int[] { 1, 2, 3, 4, 7, 11, 29, 5000 }) {
+ JsonGenerator jgen;
+
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ jgen = f.createGenerator(bytes);
+ jgen.writeStartArray();
+ InputStream data = new ThrottledInputStream(INPUT, chunkSize);
+ jgen.writeBinary(data, INPUT.length);
+ jgen.writeEndArray();
+ jgen.close();
+
+ JsonParser jp = f.createParser(bytes.toByteArray());
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, jp.nextToken());
+ byte[] b = jp.getBinaryValue();
+ Assert.assertArrayEquals(INPUT, b);
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBufferRecycle.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBufferRecycle.java
new file mode 100644
index 0000000..19bb663
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorBufferRecycle.java
@@ -0,0 +1,76 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.util.*;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+/* Test based on kimchy's issue (see https://gist.github.com/853232);
+ * exhibits an issue with buffer recycling.
+ */
+public class TestGeneratorBufferRecycle extends SmileTestBase
+{
+ public void testMaps() throws Exception
+ {
+ SmileFactory factory = new SmileFactory();
+
+ Map<?,?> props1 = buildMap("", 65);
+ Map<?,?> props2 = buildMap("", 1);
+
+ writeMapAndParse(factory, props1);
+ writeMapAndParse(factory, props2);
+ writeMapAndParse(factory, props1);
+ writeMapAndParse(factory, props2);
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private static void writeMapAndParse(SmileFactory factory, Map<?,?> map) throws Exception {
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+
+ // generate
+ JsonGenerator generator = factory.createGenerator(os);
+ writeMap(generator, map);
+ generator.close();
+
+ // parse
+ JsonParser parser = factory.createParser(os.toByteArray());
+ while (parser.nextToken() != null) {
+
+ }
+ }
+
+ private static Map<?,?> buildMap(String prefix, int size) {
+ HashMap<String,String> props = new HashMap<String, String>();
+ for (int it = 0; it < size; it++) {
+ String key = prefix + "prop_" + it;
+ props.put(key, "a");
+ }
+ return props;
+ }
+
+
+ // A sample utility to write a map
+
+ public static void writeMap(JsonGenerator gen, Map<?,?> map) throws IOException {
+ gen.writeStartObject();
+
+ for (Map.Entry<?,?> entry : map.entrySet()) {
+ gen.writeFieldName((String) entry.getKey());
+ Object value = entry.getValue();
+ if (value == null) {
+ gen.writeNull();
+ } else {
+ gen.writeString(value.toString());
+ }
+ }
+
+ gen.writeEndObject();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorLongStrings.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorLongStrings.java
new file mode 100644
index 0000000..8c2c1ac
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorLongStrings.java
@@ -0,0 +1,94 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.util.*;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+
+public class TestGeneratorLongStrings extends SmileTestBase
+{
+ final static int DOC_LEN = 2000000; // 2 meg test doc
+
+ public void testLongWithMultiBytes() throws Exception
+ {
+ SmileFactory f = new SmileFactory();
+ ArrayList<String> strings = new ArrayList<String>();
+ Random rnd = new Random(123);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream(DOC_LEN);
+ SmileGenerator gen = f.createGenerator(out);
+ gen.writeStartArray();
+
+ // Let's create 1M doc, first using Strings
+ while (out.size() < (DOC_LEN - 10000)) {
+ String str = generateString(5000, rnd);
+ strings.add(str);
+ gen.writeString(str);
+ }
+ gen.writeEndArray();
+ gen.close();
+ // Written ok; let's try parsing then
+ _verifyStrings(f, out.toByteArray(), strings);
+
+ // Then same with char[]
+ out = new ByteArrayOutputStream(DOC_LEN);
+ gen = f.createGenerator(out);
+ gen.writeStartArray();
+
+ // Let's create 1M doc, first using Strings
+ for (int i = 0, len = strings.size(); i < len; ++i) {
+ char[] ch = strings.get(i).toCharArray();
+ gen.writeString(ch, 0, ch.length);
+ }
+ gen.writeEndArray();
+ gen.close();
+ _verifyStrings(f, out.toByteArray(), strings);
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ protected String generateString(int length, Random rnd) throws Exception
+ {
+ StringBuilder sw = new StringBuilder(length+10);
+ do {
+ // First, add 7 ascii characters
+ int num = 4 + (rnd.nextInt() & 7);
+ while (--num >= 0) {
+ sw.append((char) ('A' + num));
+ }
+ // Then a unicode char of 2, 3 or 4 bytes long
+ switch (rnd.nextInt() % 3) {
+ case 0:
+ sw.append((char) (256 + rnd.nextInt() & 511));
+ break;
+ case 1:
+ sw.append((char) (2048 + rnd.nextInt() & 4095));
+ break;
+ default:
+ sw.append((char) (65536 + rnd.nextInt() & 0x3FFF));
+ break;
+ }
+ } while (sw.length() < length);
+ return sw.toString();
+ }
+
+ private void _verifyStrings(JsonFactory f, byte[] input, List<String> strings)
+ throws IOException
+ {
+ JsonParser jp = f.createParser(input);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ for (int i = 0, len = strings.size(); i < len; ++i) {
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals(strings.get(i), jp.getText());
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ jp.close();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorNumbers.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorNumbers.java
new file mode 100644
index 0000000..d9dc240
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorNumbers.java
@@ -0,0 +1,131 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.ByteArrayOutputStream;
+
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileUtil;
+
+public class TestGeneratorNumbers
+ extends SmileTestBase
+{
+ public void testSmallInts() throws Exception
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeNumber(3);
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte) (0xC0 + SmileUtil.zigzagEncode(3)));
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(0);
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte) (0xC0 + SmileUtil.zigzagEncode(0)));
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(-6);
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte) (0xC0 + SmileUtil.zigzagEncode(-6)));
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(15);
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte) (0xC0 + SmileUtil.zigzagEncode(15)));
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(-16);
+ gen.close();
+ _verifyBytes(out.toByteArray(), (byte) (0xC0 + SmileUtil.zigzagEncode(-16)));
+ }
+
+ public void testOtherInts() throws Exception
+ {
+ // beyond tiny ints, 6-bit values take 2 bytes
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeNumber(16);
+ gen.close();
+ assertEquals(2, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(-17);
+ gen.close();
+ assertEquals(2, out.toByteArray().length);
+
+ // and up to 13-bit values take 3 bytes
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(0xFFF);
+ gen.close();
+ assertEquals(3, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(-4096);
+ gen.close();
+ assertEquals(3, out.toByteArray().length);
+
+ // up to 20, 4 bytes... and so forth
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(0x1000);
+ gen.close();
+ assertEquals(4, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(500000);
+ gen.close();
+ assertEquals(4, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(Integer.MAX_VALUE);
+ gen.close();
+ assertEquals(6, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(Integer.MIN_VALUE);
+ gen.close();
+ assertEquals(6, out.toByteArray().length);
+
+ // up to longest ones, taking 11 bytes
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(Long.MAX_VALUE);
+ gen.close();
+ assertEquals(11, out.toByteArray().length);
+
+ out = new ByteArrayOutputStream();
+ gen = smileGenerator(out, false);
+ gen.writeNumber(Long.MIN_VALUE);
+ gen.close();
+ assertEquals(11, out.toByteArray().length);
+ }
+
+ public void testFloats() throws Exception
+ {
+ // float length is fixed, 6 bytes
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeNumber(0.125f);
+ gen.close();
+ assertEquals(6, out.toByteArray().length);
+ }
+
+ public void testDoubles() throws Exception
+ {
+ // double length is fixed, 11 bytes
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeNumber(0.125);
+ gen.close();
+ assertEquals(11, out.toByteArray().length);
+ }
+
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorSymbols.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorSymbols.java
new file mode 100644
index 0000000..ac523ef
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorSymbols.java
@@ -0,0 +1,241 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.ByteArrayOutputStream;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+
+public class TestGeneratorSymbols extends SmileTestBase
+{
+ /**
+ * Simple test to verify that second reference will not output new String, but
+ * rather references one output earlier.
+ */
+ public void testSharedNameSimple() throws Exception
+ {
+ // false, no header (or frame marker)
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartArray();
+ gen.writeStartObject();
+ gen.writeNumberField("abc", 1);
+ gen.writeEndObject();
+ gen.writeStartObject();
+ gen.writeNumberField("abc", 2);
+ gen.writeEndObject();
+ gen.writeEndArray();
+ gen.close();
+ byte[] result = out.toByteArray();
+ assertEquals(13, result.length);
+ }
+
+ // same as above, but with name >= 64 characters
+ public void testSharedNameSimpleLong() throws Exception
+ {
+ String digits = "01234567899";
+
+ // Base is 76 chars; loop over couple of shorter ones too
+
+ final String LONG_NAME = "a"+digits+"b"+digits+"c"+digits+"d"+digits+"e"+digits+"f"+digits+"ABCD";
+
+ for (int i = 0; i < 4; ++i) {
+ int strLen = LONG_NAME.length() - i;
+ String field = LONG_NAME.substring(0, strLen);
+ // false, no header (or frame marker)
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator gen = smileGenerator(out, false);
+ gen.writeStartArray();
+ gen.writeStartObject();
+ gen.writeNumberField(field, 1);
+ gen.writeEndObject();
+ gen.writeStartObject();
+ gen.writeNumberField(field, 2);
+ gen.writeEndObject();
+ gen.writeEndArray();
+ gen.close();
+ byte[] result = out.toByteArray();
+ assertEquals(11 + field.length(), result.length);
+
+ // better also parse it back...
+ JsonParser parser = _smileParser(result);
+ assertToken(JsonToken.START_ARRAY, parser.nextToken());
+
+ assertToken(JsonToken.START_OBJECT, parser.nextToken());
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals(field, parser.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, parser.nextToken());
+ assertEquals(1, parser.getIntValue());
+ assertToken(JsonToken.END_OBJECT, parser.nextToken());
+
+ assertToken(JsonToken.START_OBJECT, parser.nextToken());
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals(field, parser.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, parser.nextToken());
+ assertEquals(2, parser.getIntValue());
+ assertToken(JsonToken.END_OBJECT, parser.nextToken());
+
+ assertToken(JsonToken.END_ARRAY, parser.nextToken());
+ }
+ }
+
+ public void testLongNamesNonShared() throws Exception
+ {
+ _testLongNames(false);
+ }
+
+ public void testLongNamesShared() throws Exception
+ {
+ _testLongNames(true);
+ }
+
+ // [Issue#8] Test by: M. Tarik Yurt / mtyurt at gmail.com
+ public void testExpandSeenNames() throws Exception
+ {
+ byte[] data = _smileDoc("{\"a1\":null,\"a2\":null,\"a3\":null,\"a4\":null,\"a5\":null,\"a6\":null,\"a7\":null,\"a8\":null," +
+ "\"a9\":null,\"a10\":null,\"a11\":null,\"a12\":null,\"a13\":null,\"a14\":null,\"a15\":null,\"a16\":null,\"a17\":null,\"a18\":null," +
+ "\"a19\":null,\"a20\":null,\"a21\":null,\"a22\":null,\"a23\":null,\"a24\":null,\"a25\":null,\"a26\":null,\"a27\":null,\"a28\":null,\"a29\":null," +
+ "\"a30\":null,\"a31\":null,\"a32\":null,\"a33\":null,\"a34\":null,\"a35\":null,\"a36\":null,\"a37\":null,\"a38\":null,\"a39\":null,\"a40\":null," +
+ "\"a41\":null,\"a42\":null,\"a43\":null,\"a44\":null,\"a45\":null,\"a46\":null,\"a47\":null,\"a48\":null,\"a49\":null,\"a50\":null,\"a51\":null," +
+ "\"a52\":null,\"a53\":null,\"a54\":null,\"a55\":null,\"a56\":null,\"a57\":null,\"a58\":null,\"a59\":null,\"a60\":null,\"a61\":null,\"a62\":null," +
+ "\"a63\":null,\"a64\":null,"+
+ "\"a65\":{\"a32\":null}}", false);
+ /*
+ * {@code "a54".hashCode() & 63} has same value as {@code "a32".hashCode() & 63}
+ * "a32" is the next node of "a54" before expanding.
+ * 33: Null token
+ * -6: Start object token
+ * -5: End object token
+ */
+ String expectedResult = "-6,-127,97,49,33,-127,97,50,33,-127,97,51,33,-127,97,52,33,-127,97,53,33,-127,97,54,33,-127,97,55,33,-127,97,56,33,-127,97,57,33," +
+ "-126,97,49,48,33,-126,97,49,49,33,-126,97,49,50,33,-126,97,49,51,33,-126,97,49,52,33,-126,97,49,53,33,-126,97,49,54,33,-126,97,49,55,33,-126,97,49,56,33," +
+ "-126,97,49,57,33,-126,97,50,48,33,-126,97,50,49,33,-126,97,50,50,33,-126,97,50,51,33,-126,97,50,52,33,-126,97,50,53,33,-126,97,50,54,33,-126,97,50,55,33," +
+ "-126,97,50,56,33,-126,97,50,57,33,-126,97,51,48,33,-126,97,51,49,33,-126,97,51,50,33,-126,97,51,51,33,-126,97,51,52,33,-126,97,51,53,33,-126,97,51,54,33," +
+ "-126,97,51,55,33,-126,97,51,56,33,-126,97,51,57,33,-126,97,52,48,33,-126,97,52,49,33,-126,97,52,50,33,-126,97,52,51,33,-126,97,52,52,33,-126,97,52,53,33," +
+ "-126,97,52,54,33,-126,97,52,55,33,-126,97,52,56,33,-126,97,52,57,33,-126,97,53,48,33,-126,97,53,49,33,-126,97,53,50,33,-126,97,53,51,33,-126,97,53,52,33," +
+ "-126,97,53,53,33,-126,97,53,54,33,-126,97,53,55,33,-126,97,53,56,33,-126,97,53,57,33,-126,97,54,48,33,-126,97,54,49,33,-126,97,54,50,33,-126,97,54,51,33," +
+ "-126,97,54,52,33,"+
+ // "a65":{"a32":null}} :
+ "-126,97,54,53,-6,95,33,-5,-5";
+ /*
+ * First "a32" is encoded as follows: -126,97,51,50
+ * Second one should be referenced: 95
+ */
+ assertEquals(expectedResult,_dataToString(data));
+ }
+
+ // [Issue#8] Test by: M. Tarik Yurt / mtyurt at gmail.com
+ public void testExpandSeenStringValues() throws Exception
+ {
+ String json = "{\"a1\":\"v1\",\"a2\":\"v2\",\"a3\":\"v3\",\"a4\":\"v4\",\"a5\":\"v5\",\"a6\":\"v6\",\"a7\":\"v7\",\"a8\":\"v8\"," +
+ "\"a9\":\"v9\",\"a10\":\"v10\",\"a11\":\"v11\",\"a12\":\"v12\",\"a13\":\"v13\",\"a14\":\"v14\",\"a15\":\"v15\",\"a16\":\"v16\",\"a17\":\"v17\",\"a18\":\"v18\"," +
+ "\"a19\":\"v19\",\"a20\":\"v20\",\"a21\":\"v21\",\"a22\":\"v22\",\"a23\":\"v23\",\"a24\":\"v24\",\"a25\":\"v25\",\"a26\":\"v26\",\"a27\":\"v27\",\"a28\":\"v28\",\"a29\":\"v29\"," +
+ "\"a30\":\"v30\",\"a31\":\"v31\",\"a32\":\"v32\",\"a33\":\"v33\",\"a34\":\"v34\",\"a35\":\"v35\",\"a36\":\"v36\",\"a37\":\"v37\",\"a38\":\"v38\",\"a39\":\"v39\",\"a40\":\"v40\"," +
+ "\"a41\":\"v41\",\"a42\":\"v42\",\"a43\":\"v43\",\"a44\":\"v44\",\"a45\":\"v45\",\"a46\":\"v46\",\"a47\":\"v47\",\"a48\":\"v48\",\"a49\":\"v49\",\"a50\":\"v50\",\"a51\":\"v51\"," +
+ "\"a52\":\"v52\",\"a53\":\"v53\",\"a54\":\"v54\",\"a55\":\"v55\",\"a56\":\"v56\",\"a57\":\"v57\",\"a58\":\"v58\",\"a59\":\"v59\",\"a60\":\"v60\",\"a61\":\"v61\",\"a62\":\"v62\"," +
+ "\"a63\":\"v63\",\"a64\":\"v64\",\"a65\":\"v65\",\"a66\":\"v30\"}";
+ /*
+ * {@code "v52".hashCode() & 63} has same value as {@code "v30".hashCode() & 63}
+ * "v30" is next node of "v52" before expanding.
+ */
+ //Enable string value sharing
+ JsonFactory jf = new JsonFactory();
+ JsonParser jp = jf.createParser(json);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileFactory sf = new SmileFactory();
+
+ sf.configure(SmileGenerator.Feature.WRITE_HEADER, true);
+ sf.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES,true);
+ sf.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES,true);
+ JsonGenerator jg = sf.createGenerator(out, null);
+
+ while (jp.nextToken() != null) {
+ jg.copyCurrentEvent(jp);
+ }
+ jp.close();
+ jg.close();
+ /*
+ * -126,-127: Tiny key string token with length
+ * 65,66: Tiny value string token with length
+ * 97: 'a'
+ * -6: Start object token
+ * -5: End object token
+ */
+ String expectedResult = "58,41,10,3,-6,-127,97,49,65,118,49,-127,97,50,65,118,50,-127,97,51,65,118,51,-127,97,52,65,118,52,-127,97,53,65,118,53,-127,97,54,65,118,54," +
+ "-127,97,55,65,118,55,-127,97,56,65,118,56,-127,97,57,65,118,57,-126,97,49,48,66,118,49,48,-126,97,49,49,66,118,49,49,-126,97,49,50,66,118,49,50," +
+ "-126,97,49,51,66,118,49,51,-126,97,49,52,66,118,49,52,-126,97,49,53,66,118,49,53,-126,97,49,54,66,118,49,54,-126,97,49,55,66,118,49,55," +
+ "-126,97,49,56,66,118,49,56,-126,97,49,57,66,118,49,57,-126,97,50,48,66,118,50,48,-126,97,50,49,66,118,50,49,-126,97,50,50,66,118,50,50," +
+ "-126,97,50,51,66,118,50,51,-126,97,50,52,66,118,50,52,-126,97,50,53,66,118,50,53,-126,97,50,54,66,118,50,54,-126,97,50,55,66,118,50,55," +
+ "-126,97,50,56,66,118,50,56,-126,97,50,57,66,118,50,57,-126,97,51,48," +
+ "66,118,51,48," + //Here is first "v30"
+ "-126,97,51,49,66,118,51,49,-126,97,51,50,66,118,51,50," +
+ "-126,97,51,51,66,118,51,51,-126,97,51,52,66,118,51,52,-126,97,51,53,66,118,51,53,-126,97,51,54,66,118,51,54,-126,97,51,55,66,118,51,55," +
+ "-126,97,51,56,66,118,51,56,-126,97,51,57,66,118,51,57,-126,97,52,48,66,118,52,48,-126,97,52,49,66,118,52,49,-126,97,52,50,66,118,52,50," +
+ "-126,97,52,51,66,118,52,51,-126,97,52,52,66,118,52,52,-126,97,52,53,66,118,52,53,-126,97,52,54,66,118,52,54,-126,97,52,55,66,118,52,55," +
+ "-126,97,52,56,66,118,52,56,-126,97,52,57,66,118,52,57,-126,97,53,48,66,118,53,48,-126,97,53,49,66,118,53,49,-126,97,53,50,66,118,53,50," +
+ "-126,97,53,51,66,118,53,51,-126,97,53,52,66,118,53,52,-126,97,53,53,66,118,53,53,-126,97,53,54,66,118,53,54,-126,97,53,55,66,118,53,55," +
+ "-126,97,53,56,66,118,53,56,-126,97,53,57,66,118,53,57,-126,97,54,48,66,118,54,48,-126,97,54,49,66,118,54,49,-126,97,54,50,66,118,54,50," +
+ "-126,97,54,51,66,118,54,51,-126,97,54,52,66,118,54,52,-126,97,54,53,66,118,54,53,-126,97,54,54," +
+
+ //The second "v30"
+ // broken version would be:
+ //"66,118,51,48," +
+ // and correct one:
+ "30,"+
+ "-5";
+ /* First "v30" is encoded as follows: 66,118,51,48
+ * Second one should be referenced: 30
+ * But in this example, because this part is not fixed, it's encoded again: 66,118,51,48
+ */
+ assertEquals(expectedResult,_dataToString(out.toByteArray()));
+ }
+
+ /*
+ /**********************************************************
+ /* Secondary methods
+ /**********************************************************
+ */
+
+ // For issue [JACKSON-552]
+ public void _testLongNames(boolean shareNames) throws Exception
+ {
+ // 68 bytes long (on boundary)
+ final String FIELD_NAME = "dossier.domaine.supportsDeclaratifsForES.SupportDeclaratif.reference";
+ final String VALUE = "11111";
+
+ SmileFactory factory = new SmileFactory();
+ factory.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, shareNames);
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonGenerator gen = factory.createGenerator(os);
+ gen.writeStartObject();
+ gen.writeObjectFieldStart("query");
+ gen.writeStringField(FIELD_NAME, VALUE);
+ gen.writeEndObject();
+ gen.writeEndObject();
+ gen.close();
+
+ JsonParser parser = factory.createParser(os.toByteArray());
+ assertNull(parser.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, parser.nextToken());
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals("query", parser.getCurrentName());
+ assertToken(JsonToken.START_OBJECT, parser.nextToken());
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals(FIELD_NAME, parser.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, parser.nextToken());
+ assertEquals(VALUE, parser.getText());
+ assertToken(JsonToken.END_OBJECT, parser.nextToken());
+ assertToken(JsonToken.END_OBJECT, parser.nextToken());
+ }
+
+ private String _dataToString(byte[] data){
+ StringBuilder sb = new StringBuilder();
+ for(byte b:data){
+ sb.append(b).append(",");
+ }
+ sb.deleteCharAt(sb.length()-1);
+ return sb.toString();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithRawUtf8.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithRawUtf8.java
new file mode 100644
index 0000000..eb9fdf9
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithRawUtf8.java
@@ -0,0 +1,225 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+/**
+ * Test to verify handling of "raw String value" write methods that by-pass
+ * most encoding steps, for potential higher output speed (in cases where
+ * input naturally comes as UTF-8 encoded byte arrays).
+ *
+ * @since 1.7
+ */
+public class TestGeneratorWithRawUtf8 extends SmileTestBase
+{
+ public void testUtf8RawStrings() throws Exception
+ {
+ // Let's create set of Strings to output; no ctrl chars as we do raw
+ List<byte[]> strings = generateStrings(new Random(28), 750000, false);
+ ByteArrayOutputStream out = new ByteArrayOutputStream(16000);
+ SmileFactory jf = new SmileFactory();
+ JsonGenerator jgen = jf.createGenerator(out, JsonEncoding.UTF8);
+ jgen.writeStartArray();
+ for (byte[] str : strings) {
+ jgen.writeRawUTF8String(str, 0, str.length);
+ }
+ jgen.writeEndArray();
+ jgen.close();
+ byte[] json = out.toByteArray();
+
+ // Ok: let's verify that stuff was written out ok
+ JsonParser jp = jf.createParser(json);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ for (byte[] inputBytes : strings) {
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ String string = jp.getText();
+ byte[] outputBytes = string.getBytes("UTF-8");
+ assertEquals(inputBytes.length, outputBytes.length);
+ assertArrayEquals(inputBytes, outputBytes);
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+
+ /**
+ * Unit test for "JsonGenerator.writeUTF8String()", which needs
+ * to handle escaping properly
+ */
+ public void testUtf8StringsWithEscaping() throws Exception
+ {
+ // Let's create set of Strings to output; do include control chars too:
+ List<byte[]> strings = generateStrings(new Random(28), 720000, true);
+ ByteArrayOutputStream out = new ByteArrayOutputStream(16000);
+ SmileFactory jf = new SmileFactory();
+ JsonGenerator jgen = jf.createGenerator(out, JsonEncoding.UTF8);
+ jgen.writeStartArray();
+ for (byte[] str : strings) {
+ jgen.writeUTF8String(str, 0, str.length);
+ }
+ jgen.writeEndArray();
+ jgen.close();
+ byte[] json = out.toByteArray();
+
+ // Ok: let's verify that stuff was written out ok
+ JsonParser jp = jf.createParser(json);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ for (byte[] inputBytes : strings) {
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ String string = jp.getText();
+ byte[] outputBytes = string.getBytes("UTF-8");
+ assertEquals(inputBytes.length, outputBytes.length);
+ assertArrayEquals(inputBytes, outputBytes);
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+
+ /**
+ * Test to point out an issue with "raw" UTF-8 encoding
+ *
+ * @author David Yu
+ */
+ public void testIssue492() throws Exception
+ {
+ doTestIssue492(false);
+ doTestIssue492(true);
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void doTestIssue492(boolean asUtf8String) throws Exception
+ {
+ SmileFactory factory = new SmileFactory();
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ SmileGenerator generator = factory.createGenerator(out);
+
+ generator.writeStartObject();
+
+ generator.writeFieldName("name");
+
+ if(asUtf8String)
+ {
+ byte[] text = "PojoFoo".getBytes("ASCII");
+ generator.writeUTF8String(text, 0, text.length);
+ }
+ else
+ {
+ generator.writeString("PojoFoo");
+ }
+
+ generator.writeFieldName("collection");
+
+ generator.writeStartObject();
+
+ generator.writeFieldName("v");
+
+ generator.writeStartArray();
+
+ if(asUtf8String)
+ {
+ byte[] text = "1".getBytes("ASCII");
+ generator.writeUTF8String(text, 0, text.length);
+ }
+ else
+ {
+ generator.writeString("1");
+ }
+
+ generator.writeEndArray();
+
+ generator.writeEndObject();
+
+ generator.writeEndObject();
+
+ generator.close();
+
+ byte[] data = out.toByteArray();
+
+ ByteArrayInputStream in = new ByteArrayInputStream(data);
+ SmileParser parser = factory.createParser(in);
+
+ assertToken(parser.nextToken(), JsonToken.START_OBJECT);
+
+ assertToken(parser.nextToken(), JsonToken.FIELD_NAME);
+ assertEquals(parser.getCurrentName(), "name");
+ assertToken(parser.nextToken(), JsonToken.VALUE_STRING);
+ assertEquals(parser.getText(), "PojoFoo");
+
+ assertToken(parser.nextToken(), JsonToken.FIELD_NAME);
+ assertEquals(parser.getCurrentName(), "collection");
+ assertToken(parser.nextToken(), JsonToken.START_OBJECT);
+
+ assertToken(parser.nextToken(), JsonToken.FIELD_NAME);
+ assertEquals("Should have property with name 'v'", parser.getCurrentName(), "v");
+ assertToken(parser.nextToken(), JsonToken.START_ARRAY);
+
+ assertToken(parser.nextToken(), JsonToken.VALUE_STRING);
+ assertEquals("Should get String value '1'", parser.getText(), "1");
+
+ assertToken(parser.nextToken(), JsonToken.END_ARRAY);
+ assertToken(parser.nextToken(), JsonToken.END_OBJECT);
+
+
+ assertToken(parser.nextToken(), JsonToken.END_OBJECT);
+ parser.close();
+ }
+
+ private List<byte[]> generateStrings(Random rnd, int totalLength, boolean includeCtrlChars)
+ throws IOException
+ {
+ ArrayList<byte[]> strings = new ArrayList<byte[]>();
+ do {
+ int len = 2;
+ int bits = rnd.nextInt(14);
+ while (--bits >= 0) {
+ len += len;
+ }
+ len = 1 + ((len + len) / 3);
+ String str = generateString(rnd, len, includeCtrlChars);
+ byte[] bytes = str.getBytes("UTF-8");
+ strings.add(bytes);
+ totalLength -= bytes.length;
+ } while (totalLength > 0);
+ return strings;
+ }
+
+ private String generateString(Random rnd, int length, boolean includeCtrlChars)
+ {
+ StringBuilder sb = new StringBuilder(length);
+ do {
+ int i;
+ switch (rnd.nextInt(3)) {
+ case 0: // 3 byte one
+ i = 2048 + rnd.nextInt(16383);
+ break;
+ case 1: // 2 byte
+ i = 128 + rnd.nextInt(1024);
+ break;
+ default: // ASCII
+ i = rnd.nextInt(192);
+ if (!includeCtrlChars) {
+ i += 32;
+ // but also need to avoid backslash, double-quote
+ if (i == '\\' || i == '"') {
+ i = '@'; // just arbitrary choice
+ }
+ }
+ }
+ sb.append((char) i);
+ } while (sb.length() < length);
+ return sb.toString();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithSerializedString.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithSerializedString.java
new file mode 100644
index 0000000..9eedaa6
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestGeneratorWithSerializedString.java
@@ -0,0 +1,86 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.ByteArrayOutputStream;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.io.SerializedString;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+public class TestGeneratorWithSerializedString extends SmileTestBase
+{
+ final static String NAME_WITH_QUOTES = "\"name\"";
+ final static String NAME_WITH_LATIN1 = "P\u00f6ll\u00f6";
+
+ private final SerializedString quotedName = new SerializedString(NAME_WITH_QUOTES);
+ private final SerializedString latin1Name = new SerializedString(NAME_WITH_LATIN1);
+
+ public void testSimple() throws Exception
+ {
+ SmileFactory sf = new SmileFactory();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ JsonGenerator jgen = sf.createGenerator(out);
+ _writeSimple(jgen);
+ jgen.close();
+ byte[] smileB = out.toByteArray();
+ _verifySimple(sf.createParser(smileB));
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void _writeSimple(JsonGenerator jgen) throws Exception
+ {
+ // Let's just write array of 2 objects
+ jgen.writeStartArray();
+
+ jgen.writeStartObject();
+ jgen.writeFieldName(quotedName);
+ jgen.writeString("a");
+ jgen.writeFieldName(latin1Name);
+ jgen.writeString("b");
+ jgen.writeEndObject();
+
+ jgen.writeStartObject();
+ jgen.writeFieldName(latin1Name);
+ jgen.writeString("c");
+ jgen.writeFieldName(quotedName);
+ jgen.writeString("d");
+ jgen.writeEndObject();
+
+ jgen.writeEndArray();
+ }
+
+ private void _verifySimple(JsonParser jp) throws Exception
+ {
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(NAME_WITH_QUOTES, jp.getText());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("a", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(NAME_WITH_LATIN1, jp.getText());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("b", jp.getText());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(NAME_WITH_LATIN1, jp.getText());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("c", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(NAME_WITH_QUOTES, jp.getText());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("d", jp.getText());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ assertNull(jp.nextToken());
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestMapper.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestMapper.java
new file mode 100644
index 0000000..212c77f
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestMapper.java
@@ -0,0 +1,42 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class TestMapper extends SmileTestBase
+{
+ static class BytesBean {
+ public byte[] bytes;
+
+ public BytesBean() { }
+ public BytesBean(byte[] b) { bytes = b; }
+ }
+
+ // [JACKSON-733]
+ public void testBinary() throws IOException
+ {
+ byte[] input = new byte[] { 1, 2, 3, -1, 8, 0, 42 };
+ ObjectMapper mapper = smileMapper();
+ byte[] smile = mapper.writeValueAsBytes(new BytesBean(input));
+ BytesBean result = mapper.readValue(smile, BytesBean.class);
+
+ assertNotNull(result.bytes);
+ Assert.assertArrayEquals(input, result.bytes);
+ }
+
+ // @since 2.1
+ public void testCopy() throws IOException
+ {
+ ObjectMapper mapper1 = smileMapper();
+ ObjectMapper mapper2 = mapper1.copy();
+
+ assertNotSame(mapper1, mapper2);
+ assertNotSame(mapper1.getFactory(), mapper2.getFactory());
+ assertEquals(SmileFactory.class, mapper2.getFactory().getClass());
+ }
+}
+
+
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParser.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParser.java
new file mode 100644
index 0000000..7049741
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParser.java
@@ -0,0 +1,558 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileConstants;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+public class TestParser
+ extends SmileTestBase
+{
+ // Unit tests for verifying that if header/signature is required,
+ // lacking it is fatal
+ public void testMandatoryHeader() throws IOException
+ {
+ // first test failing case
+ byte[] data = _smileDoc("[ null ]", false);
+ try {
+ _smileParser(data, true);
+ fail("Should have gotten exception for missing header");
+ } catch (Exception e) {
+ verifyException(e, "does not start with Smile format header");
+ }
+
+ // and then test passing one
+ SmileParser p = _smileParser(data, false);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertNull(p.nextToken());
+ }
+
+ public void testSimple() throws IOException
+ {
+ byte[] data = _smileDoc("[ true, null, false ]");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.VALUE_FALSE, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.getCurrentName());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ public void testArrayWithString() throws IOException
+ {
+ byte[] data = _smileDoc("[ \"abc\" ]");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("abc", p.getText());
+ assertEquals(0, p.getTextOffset());
+ assertEquals(3, p.getTextLength());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+
+ public void testEmptyStrings() throws IOException
+ {
+ // first, empty key
+ byte[] data = _smileDoc("{ \"\":true }");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("", p.getCurrentName());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // then empty value
+ data = _smileDoc("{ \"abc\":\"\" }");
+ p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("abc", p.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("", p.getText());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // and combinations
+ data = _smileDoc("{ \"\":\"\", \"\":\"\" }");
+ p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("", p.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("", p.getText());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("", p.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("", p.getText());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ // Test for ASCII String values longer than 64 bytes; separate
+ // since handling differs
+ public void testLongAsciiString() throws IOException
+ {
+ final String DIGITS = "1234567890";
+ String LONG = DIGITS + DIGITS + DIGITS + DIGITS;
+ LONG = LONG + LONG + LONG + LONG;
+ byte[] data = _smileDoc(quote(LONG));
+
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(LONG, p.getText());
+ assertNull(p.nextToken());
+ }
+
+ //Test for non-ASCII String values longer than 64 bytes; separate
+ // since handling differs
+ public void testLongUnicodeString() throws IOException
+ {
+ final String DIGITS = "1234567890";
+ final String UNIC = "\u00F06"; // o with umlauts
+ String LONG = DIGITS + UNIC + DIGITS + UNIC + UNIC + DIGITS + DIGITS;
+ LONG = LONG + LONG + LONG;
+ byte[] data = _smileDoc(quote(LONG));
+
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(LONG, p.getText());
+ assertNull(p.nextToken());
+ }
+
+ public void testTrivialObject() throws IOException
+ {
+ byte[] data = _smileDoc("{\"abc\":13}");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("abc", p.getCurrentName());
+ assertEquals("abc", p.getText());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(13, p.getIntValue());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ }
+
+ public void testSimpleObject() throws IOException
+ {
+ byte[] data = _smileDoc("{\"a\":8, \"b\" : [ true ], \"c\" : { }, \"d\":{\"e\":null}}");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.getCurrentName());
+ assertEquals("a", p.getText());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(8, p.getIntValue());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.getCurrentName());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("c", p.getCurrentName());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("d", p.getCurrentName());
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("e", p.getCurrentName());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ p.close();
+ }
+
+ public void testNestedObject() throws IOException
+ {
+ byte[] data = _smileDoc("[{\"a\":{\"b\":[1]}}]");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken()); // a
+ assertEquals("a", p.getCurrentName());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken()); // b
+ assertEquals("b", p.getCurrentName());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertEquals("b", p.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ }
+
+ public void testJsonSampleDoc() throws IOException
+ {
+ byte[] data = _smileDoc(SAMPLE_DOC_JSON_SPEC);
+ verifyJsonSpecSampleDoc(_smileParser(data), true);
+ }
+
+ public void testUnicodeStringValues() throws IOException
+ {
+ String uc = "\u00f6stl. v. Greenwich \u3333?";
+ byte[] data = _smileDoc("[" +quote(uc)+"]");
+
+ // First, just skipping
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // Then accessing data
+ p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(uc, p.getText());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // and then let's create longer text segment as well
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < 200) {
+ sb.append(uc);
+ }
+ final String longer = sb.toString();
+ data = _smileDoc("["+quote(longer)+"]");
+
+ // Ok once again, first skipping, then accessing
+ p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(longer, p.getText());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ public void testUnicodePropertyNames() throws IOException
+ {
+ String uc = "\u00f6stl. v. Greenwich \u3333";
+ byte[] data = _smileDoc("{" +quote(uc)+":true}");
+
+ // First, just skipping
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // Then accessing data
+ p = _smileParser(data);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals(uc, p.getCurrentName());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ // Simple test to verify that byte 0 is not used (an implementation
+ // might mistakenly consider it a string value reference)
+ public void testInvalidByte() throws IOException
+ {
+ byte[] data = new byte[] { SmileConstants.TOKEN_LITERAL_START_ARRAY,
+ (byte) SmileConstants.TOKEN_PREFIX_SHARED_STRING_SHORT,
+ (byte) SmileConstants.TOKEN_LITERAL_END_ARRAY
+ };
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ // And now should get an error
+ try {
+ JsonToken t = p.nextToken();
+ fail("Expected parse error, got: "+t);
+ } catch (IOException e) {
+ verifyException(e, "Invalid token byte 0x00");
+ }
+ }
+
+ // [JACKSON-629]
+ public void testNameBoundary() throws IOException
+ {
+ SmileFactory f = smileFactory(true, true, false);
+ // let's create 3 meg docs
+ final int LEN = 3 * 1000 * 1000;
+ final String FIELD = "field01"; // important: 7 chars
+
+ for (int offset = 0; offset < 12; ++offset) {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(LEN);
+ // To trigger boundary condition, need to shuffle stuff around a bit...
+ for (int i = 0; i < offset; ++i) {
+ bytes.write(0);
+ }
+
+ // force back-refs off, easier to trigger problem
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, false);
+ SmileGenerator gen = f.createGenerator(bytes);
+
+ int count = 0;
+ do {
+ gen.writeStartObject();
+ // importa
+ gen.writeNumberField(FIELD, count % 17);
+ gen.writeEndObject();
+ ++count;
+ } while (bytes.size() < (LEN - 100));
+ gen.close();
+
+ // and then read back
+ byte[] json = bytes.toByteArray();
+ SmileParser jp = f.createParser(new ByteArrayInputStream(json, offset, json.length-offset));
+ int i = 0;
+
+ while (i < count) {
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(FIELD, jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals((i % 17), jp.getIntValue());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ ++i;
+ }
+ // and should be done now
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+ }
+
+ // [JACKSON-640]: Problem with getTextCharacters/Offset/Length
+ public void testCharacters() throws IOException
+ {
+ // ensure we are using both back-ref types
+ SmileFactory sf = new SmileFactory();
+ sf.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, true);
+ sf.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(100);
+
+ JsonGenerator jgen = sf.createGenerator(bytes);
+ jgen.writeStartArray();
+ jgen.writeStartObject();
+ jgen.writeStringField("key", "value");
+ jgen.writeEndObject();
+ jgen.writeStartObject();
+ jgen.writeStringField("key", "value");
+ jgen.writeEndObject();
+ jgen.writeEndArray();
+ jgen.close();
+
+ SmileParser p = _smileParser(bytes.toByteArray());
+
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ String str;
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ str = new String(p.getTextCharacters(), p.getTextOffset(), p.getTextLength());
+ assertEquals("key", str);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ str = new String(p.getTextCharacters(), p.getTextOffset(), p.getTextLength());
+ assertEquals("value", str);
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ str = new String(p.getTextCharacters(), p.getTextOffset(), p.getTextLength());
+ assertEquals("key", str);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ str = new String(p.getTextCharacters(), p.getTextOffset(), p.getTextLength());
+ assertEquals("value", str);
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods for use with json spec sample doc
+ /**********************************************************
+ */
+
+ protected void verifyJsonSpecSampleDoc(JsonParser jp, boolean verifyContents)
+ throws IOException
+ {
+ verifyJsonSpecSampleDoc(jp, verifyContents, true);
+ }
+
+ protected void verifyJsonSpecSampleDoc(JsonParser jp, boolean verifyContents,
+ boolean requireNumbers)
+ throws IOException
+ {
+ if (!jp.hasCurrentToken()) {
+ jp.nextToken();
+ }
+ assertToken(JsonToken.START_OBJECT, jp.getCurrentToken()); // main object
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Image'
+ if (verifyContents) {
+ verifyFieldName(jp, "Image");
+ }
+
+ assertToken(JsonToken.START_OBJECT, jp.nextToken()); // 'image' object
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Width'
+ if (verifyContents) {
+ verifyFieldName(jp, "Width");
+ }
+
+ verifyIntToken(jp.nextToken(), requireNumbers);
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_WIDTH);
+ }
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Height'
+ if (verifyContents) {
+ verifyFieldName(jp, "Height");
+ }
+
+ verifyIntToken(jp.nextToken(), requireNumbers);
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_HEIGHT);
+ }
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Title'
+ if (verifyContents) {
+ verifyFieldName(jp, "Title");
+ }
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals(SAMPLE_SPEC_VALUE_TITLE, getAndVerifyText(jp));
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Thumbnail'
+ if (verifyContents) {
+ verifyFieldName(jp, "Thumbnail");
+ }
+
+ assertToken(JsonToken.START_OBJECT, jp.nextToken()); // 'thumbnail' object
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Url'
+ if (verifyContents) {
+ verifyFieldName(jp, "Url");
+ }
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ if (verifyContents) {
+ assertEquals(SAMPLE_SPEC_VALUE_TN_URL, getAndVerifyText(jp));
+ }
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Height'
+ if (verifyContents) {
+ verifyFieldName(jp, "Height");
+ }
+ verifyIntToken(jp.nextToken(), requireNumbers);
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_TN_HEIGHT);
+ }
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'Width'
+ if (verifyContents) {
+ verifyFieldName(jp, "Width");
+ }
+ // Width value is actually a String in the example
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ if (verifyContents) {
+ assertEquals(SAMPLE_SPEC_VALUE_TN_WIDTH, getAndVerifyText(jp));
+ }
+
+ assertToken(JsonToken.END_OBJECT, jp.nextToken()); // 'thumbnail' object
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken()); // 'IDs'
+ assertToken(JsonToken.START_ARRAY, jp.nextToken()); // 'ids' array
+ verifyIntToken(jp.nextToken(), requireNumbers); // ids[0]
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_TN_ID1);
+ }
+ verifyIntToken(jp.nextToken(), requireNumbers); // ids[1]
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_TN_ID2);
+ }
+ verifyIntToken(jp.nextToken(), requireNumbers); // ids[2]
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_TN_ID3);
+ }
+ verifyIntToken(jp.nextToken(), requireNumbers); // ids[3]
+ if (verifyContents) {
+ verifyIntValue(jp, SAMPLE_SPEC_VALUE_TN_ID4);
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken()); // 'ids' array
+
+ assertToken(JsonToken.END_OBJECT, jp.nextToken()); // 'image' object
+
+ assertToken(JsonToken.END_OBJECT, jp.nextToken()); // main object
+ }
+
+ private void verifyIntToken(JsonToken t, boolean requireNumbers)
+ {
+ if (t == JsonToken.VALUE_NUMBER_INT) {
+ return;
+ }
+ if (requireNumbers) { // to get error
+ assertToken(JsonToken.VALUE_NUMBER_INT, t);
+ }
+ // if not number, must be String
+ if (t != JsonToken.VALUE_STRING) {
+ fail("Expected INT or STRING value, got "+t);
+ }
+ }
+
+ protected void verifyFieldName(JsonParser jp, String expName)
+ throws IOException
+ {
+ assertEquals(expName, jp.getText());
+ assertEquals(expName, jp.getCurrentName());
+ }
+
+ protected void verifyIntValue(JsonParser jp, long expValue)
+ throws IOException
+ {
+ // First, via textual
+ assertEquals(String.valueOf(expValue), jp.getText());
+ }
+
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserBinary.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserBinary.java
new file mode 100644
index 0000000..a6f0c99
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserBinary.java
@@ -0,0 +1,214 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import static org.junit.Assert.*;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+public class TestParserBinary
+ extends SmileTestBase
+{
+ final static int[] SIZES = new int[] {
+ 1, 2, 3, 4, 5, 6,
+ 7, 8, 12,
+ 100, 350, 1900, 6000, 19000, 65000,
+ 139000
+ };
+
+ public void testRawAsArray() throws IOException
+ {
+ _testBinaryAsArray(true);
+ }
+
+ public void test7BitAsArray() throws IOException
+ {
+ _testBinaryAsArray(false);
+ }
+
+ // Added based on [JACKSON-376]
+ public void testRawAsObject() throws IOException
+ {
+ _testBinaryAsObject(true);
+ }
+
+ // Added based on [JACKSON-376]
+ public void test7BitAsObject() throws IOException
+ {
+ _testBinaryAsObject(false);
+ }
+
+ public void testRawAsRootValue() throws IOException {
+ _testBinaryAsRoot(true);
+ }
+
+ public void test7BitAsRootValue() throws IOException {
+ _testBinaryAsRoot(false);
+ }
+
+ // [Issue-17] (streaming binary reads)
+ public void testStreamingRaw() throws IOException {
+ _testStreaming(true);
+ }
+
+ // [Issue-17] (streaming binary reads)
+ public void testStreamingEncoded() throws IOException {
+ _testStreaming(false);
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void _testBinaryAsRoot(boolean raw) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !raw);
+ for (int size : SIZES) {
+ byte[] data = _generateData(size);
+ ByteArrayOutputStream bo = new ByteArrayOutputStream(size+10);
+ SmileGenerator g = f.createGenerator(bo);
+ g.writeBinary(data);
+ g.close();
+ byte[] smile = bo.toByteArray();
+
+ // and verify
+ SmileParser p = f.createParser(smile);
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ byte[] result = p.getBinaryValue();
+ assertArrayEquals(data, result);
+ assertNull(p.nextToken());
+ p.close();
+
+ // and second time around, skipping
+ p = f.createParser(smile);
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+ }
+
+ private void _testBinaryAsArray(boolean raw) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !raw);
+ for (int size : SIZES) {
+ byte[] data = _generateData(size);
+ ByteArrayOutputStream bo = new ByteArrayOutputStream(size+10);
+ SmileGenerator g = f.createGenerator(bo);
+ g.writeStartArray();
+ g.writeBinary(data);
+ g.writeNumber(1); // just to verify there's no overrun
+ g.writeEndArray();
+ g.close();
+ byte[] smile = bo.toByteArray();
+
+ // and verify
+ SmileParser p = f.createParser(smile);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ byte[] result = p.getBinaryValue();
+ assertArrayEquals(data, result);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(1, p.getIntValue());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // and second time around, skipping
+ p = f.createParser(smile);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+ }
+
+ private void _testBinaryAsObject(boolean raw) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !raw);
+ for (int size : SIZES) {
+ byte[] data = _generateData(size);
+ ByteArrayOutputStream bo = new ByteArrayOutputStream(size+10);
+ SmileGenerator g = f.createGenerator(bo);
+ g.writeStartObject();
+ g.writeFieldName("binary");
+ g.writeBinary(data);
+ g.writeEndObject();
+ g.close();
+ byte[] smile = bo.toByteArray();
+
+ // and verify
+ SmileParser p = f.createParser(smile);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("binary", p.getCurrentName());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ byte[] result = p.getBinaryValue();
+ assertArrayEquals(data, result);
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+
+ // and second time around, skipping
+ p = f.createParser(smile);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+ }
+
+ private void _testStreaming(boolean raw) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !raw);
+ for (int size : SIZES) {
+ byte[] data = _generateData(size);
+ ByteArrayOutputStream bo = new ByteArrayOutputStream(size+10);
+ SmileGenerator g = f.createGenerator(bo);
+ g.writeStartObject();
+ g.writeFieldName("b");
+ g.writeBinary(data);
+ g.writeEndObject();
+ g.close();
+ byte[] smile = bo.toByteArray();
+
+ // and verify
+ SmileParser p = f.createParser(smile);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.getCurrentName());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken());
+ ByteArrayOutputStream result = new ByteArrayOutputStream(size);
+ int gotten = p.readBinaryValue(result);
+ assertEquals(size, gotten);
+ assertArrayEquals(data, result.toByteArray());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+ }
+
+ private byte[] _generateData(int size)
+ {
+ byte[] result = new byte[size];
+ for (int i = 0; i < size; ++i) {
+ result[i] = (byte) (i % 255);
+ }
+ return result;
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserLocation.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserLocation.java
new file mode 100644
index 0000000..194d62d
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserLocation.java
@@ -0,0 +1,60 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.*;
+
+public class TestParserLocation
+ extends SmileTestBase
+{
+ /**
+ * Basic unit test to verify that [JACKSON-] has been resolved.
+ */
+ public void testSimpleOffsets() throws IOException
+ {
+ byte[] data = _smileDoc("[ true, null, false, 511 ]", true); // true -> write header
+
+ JsonParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ JsonLocation loc = p.getCurrentLocation();
+ assertNotNull(loc);
+ // first: -1 for "not known", for character-based stuff
+ assertEquals(-1, loc.getCharOffset());
+ // column will indicate offset, so:
+ assertEquals(4, loc.getColumnNr());
+ assertEquals(-1, loc.getLineNr());
+ // but first 4 bytes are for header
+ assertEquals(4, loc.getByteOffset());
+
+ // array marker is a single byte, so:
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertEquals(5, p.getCurrentLocation().getByteOffset());
+ assertEquals(4, p.getTokenLocation().getByteOffset());
+
+ // same for true and others except for last int
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+ assertEquals(6, p.getCurrentLocation().getByteOffset());
+ assertEquals(5, p.getTokenLocation().getByteOffset());
+
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertEquals(7, p.getCurrentLocation().getByteOffset());
+ assertEquals(6, p.getTokenLocation().getByteOffset());
+
+ assertToken(JsonToken.VALUE_FALSE, p.nextToken());
+ assertEquals(8, p.getCurrentLocation().getByteOffset());
+ assertEquals(7, p.getTokenLocation().getByteOffset());
+
+ // 0x1FF takes 3 bytes (type byte, 7/6 bit segments)
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(511, p.getIntValue());
+ assertEquals(11, p.getCurrentLocation().getByteOffset());
+ assertEquals(8, p.getTokenLocation().getByteOffset());
+
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEquals(12, p.getCurrentLocation().getByteOffset());
+ assertEquals(11, p.getTokenLocation().getByteOffset());
+
+ assertNull(p.nextToken());
+ p.close();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNames.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNames.java
new file mode 100644
index 0000000..af9f7b1
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNames.java
@@ -0,0 +1,66 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.util.Random;
+
+import com.fasterxml.jackson.core.*;
+
+public class TestParserNames extends SmileTestBase
+{
+ public void testLongNames() throws IOException
+ {
+ _testWithName(generateName(5000));
+ }
+
+ public void testJsonBinForLargeObjects() throws Exception
+ {
+ StringBuilder nameBuf = new StringBuilder("longString");
+ int minLength = 9000;
+ for (int i = 1; nameBuf.length() < minLength; ++i) {
+ nameBuf.append("." + i);
+ }
+ String name = nameBuf.toString();
+ _testWithName(name);
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void _testWithName(String name) throws IOException
+ {
+ byte[] data = _smileDoc("{"+quote(name)+":13}");
+ // important: MUST use InputStream to enforce buffer boundaries!
+ SmileParser p = _smileParser(new ByteArrayInputStream(data));
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals(name, p.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(13, p.getIntValue());
+ assertEquals(name, p.getCurrentName());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+
+ private String generateName(int minLen)
+ {
+ StringBuilder sb = new StringBuilder();
+ Random rnd = new Random(123);
+ while (sb.length() < minLen) {
+ int ch = rnd.nextInt(96);
+ if (ch < 32) { // ascii (single byte)
+ sb.append((char) (48 + ch));
+ } else if (ch < 64) { // 2 byte
+ sb.append((char) (128 + ch));
+ } else { // 3 byte
+ sb.append((char) (4000 + ch));
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNumbers.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNumbers.java
new file mode 100644
index 0000000..0dbfcbe
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserNumbers.java
@@ -0,0 +1,303 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import com.fasterxml.jackson.core.*;
+
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+public class TestParserNumbers
+ extends SmileTestBase
+{
+ public void testIntsMedium() throws IOException
+ {
+ byte[] data = _smileDoc("255");
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(255, p.getIntValue());
+ assertEquals("255", p.getText());
+
+ data = _smileDoc("-999");
+ p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertEquals(-999, p.getIntValue());
+ assertEquals("-999", p.getText());
+
+ data = _smileDoc("123456789");
+ p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertEquals(123456789, p.getIntValue());
+ }
+
+ public void testMinMaxInts() throws IOException
+ {
+ byte[] data = _smileDoc(String.valueOf(Integer.MAX_VALUE));
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertEquals(Integer.MAX_VALUE, p.getIntValue());
+
+ data = _smileDoc(String.valueOf(Integer.MIN_VALUE));
+ p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertEquals(Integer.MIN_VALUE, p.getIntValue());
+ }
+
+ public void testIntsInObjectSkipping() throws IOException
+ {
+ byte[] data = _smileDoc("{\"a\":200,\"b\":200}");
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ // let's NOT access value, forcing skipping
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ // let's NOT access value, forcing skipping
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ }
+
+ public void testBorderLongs() throws IOException
+ {
+ long l = (long) Integer.MIN_VALUE - 1L;
+ byte[] data = _smileDoc(String.valueOf(l), false);
+ assertEquals(6, data.length);
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+ assertEquals(l, p.getLongValue());
+
+ l = 1L + (long) Integer.MAX_VALUE;
+ data = _smileDoc(String.valueOf(l), false);
+ assertEquals(6, data.length);
+ p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+ assertEquals(l, p.getLongValue());
+ }
+
+ public void testLongs() throws IOException
+ {
+ long l = Long.MAX_VALUE;
+ byte[] data = _smileDoc(String.valueOf(l));
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+ assertEquals(l, p.getLongValue());
+ assertEquals(String.valueOf(l), p.getText());
+
+ l = Long.MIN_VALUE;
+ data = _smileDoc(String.valueOf(l));
+ p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+ assertEquals(l, p.getLongValue());
+ assertEquals(String.valueOf(l), p.getText());
+ }
+
+ public void testArrayWithInts() throws IOException
+ {
+ byte[] data = _smileDoc("[ 1, 0, -1, 255, -999, "
+ +Integer.MIN_VALUE+","+Integer.MAX_VALUE+","
+ +Long.MIN_VALUE+", "+Long.MAX_VALUE+" ]");
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(1, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(0, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(-1, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(255, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(-999, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+ assertEquals(Integer.MIN_VALUE, p.getIntValue());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(Integer.MAX_VALUE, p.getIntValue());
+ assertEquals(JsonParser.NumberType.INT, p.getNumberType());
+
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+ assertEquals(Long.MIN_VALUE, p.getLongValue());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(Long.MAX_VALUE, p.getLongValue());
+ assertEquals(JsonParser.NumberType.LONG, p.getNumberType());
+
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+
+ public void testFloats() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ SmileGenerator g = smileGenerator(bo, false);
+ float value = 0.37f;
+ g.writeNumber(value);
+ g.close();
+ byte[] data = bo.toByteArray();
+ assertEquals(6, data.length);
+
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(JsonParser.NumberType.FLOAT, p.getNumberType());
+ assertEquals(value, p.getFloatValue());
+ }
+
+ public void testDoubles() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ SmileGenerator g = smileGenerator(bo, false);
+ double value = -12.0986;
+ g.writeNumber(value);
+ g.close();
+ byte[] data = bo.toByteArray();
+ assertEquals(11, data.length);
+
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(JsonParser.NumberType.DOUBLE, p.getNumberType());
+ assertEquals(value, p.getDoubleValue());
+ }
+
+ public void testArrayWithDoubles() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ SmileGenerator g = smileGenerator(bo, false);
+ g.writeStartArray();
+ g.writeNumber(0.1f);
+ g.writeNumber(0.333);
+ g.writeEndArray();
+ g.close();
+ byte[] data = bo.toByteArray();
+ assertEquals(19, data.length);
+
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(JsonParser.NumberType.FLOAT, p.getNumberType());
+ assertEquals(0.1f, p.getFloatValue());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(JsonParser.NumberType.DOUBLE, p.getNumberType());
+ assertEquals(0.333, p.getDoubleValue());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ }
+
+ public void testObjectWithDoubles() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ SmileGenerator g = smileGenerator(bo, false);
+ g.writeStartObject();
+ g.writeNumberField("x", 0.5);
+ g.writeNumberField("y", 0.01338);
+ g.writeEndObject();
+ g.close();
+
+ byte[] data = bo.toByteArray();
+
+ // first let's just skip
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ p.close();
+
+ // and then check data too (skip codepath distinct)
+ p = _smileParser(data);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("x", p.getText());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(0.5, p.getDoubleValue());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("y", p.getText());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(0.01338, p.getDoubleValue());
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ p.close();
+ }
+
+ public void testBigInteger() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ BigInteger in = new BigInteger(String.valueOf(Long.MIN_VALUE)+"0012575934");
+ SmileGenerator g = smileGenerator(bo, false);
+ g.writeNumber(in);
+ g.close();
+ byte[] data = bo.toByteArray();
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonParser.NumberType.BIG_INTEGER, p.getNumberType());
+ assertEquals(BigInteger.class, p.getNumberValue().getClass());
+ assertEquals(in, p.getBigIntegerValue());
+ p.close();
+
+ // second test; verify skipping works
+ bo = new ByteArrayOutputStream();
+ g = smileGenerator(bo, false);
+ g.writeStartArray();
+ g.writeNumber(in);
+ g.writeEndArray();
+ g.close();
+ data = bo.toByteArray();
+ p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ public void testBigDecimal() throws IOException
+ {
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
+ BigDecimal in = new BigDecimal("32599.00001");
+ SmileGenerator g = smileGenerator(bo, false);
+ g.writeNumber(in);
+ g.close();
+ byte[] data = bo.toByteArray();
+ SmileParser p = _smileParser(data);
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(JsonParser.NumberType.BIG_DECIMAL, p.getNumberType());
+ assertEquals(BigDecimal.class, p.getNumberValue().getClass());
+ assertEquals(in, p.getDecimalValue());
+ p.close();
+
+ // second test; verify skipping works
+ bo = new ByteArrayOutputStream();
+ g = smileGenerator(bo, false);
+ g.writeStartArray();
+ g.writeNumber(in);
+ g.writeEndArray();
+ g.close();
+ data = bo.toByteArray();
+ p = _smileParser(data);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserSymbolHandling.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserSymbolHandling.java
new file mode 100644
index 0000000..a124152
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserSymbolHandling.java
@@ -0,0 +1,564 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+import java.util.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+/**
+ * Unit tests for verifying that symbol handling works as planned, including
+ * efficient reuse of names encountered during parsing.
+ */
+public class TestParserSymbolHandling
+ extends SmileTestBase
+{
+ /*
+ /**********************************************************
+ /* Helper types, constants
+ /**********************************************************
+ */
+
+ private final static String[] SHARED_SYMBOLS = new String[] {
+ "g", "J", "v", "B", "S", "JAVA",
+ "h", "J", "LARGE",
+ "JAVA", "J", "SMALL"
+ };
+
+ static class MediaItem
+ {
+ public Content content;
+ public Image[] images;
+ }
+
+ public enum Size { SMALL, LARGE; }
+ public enum Player { JAVA, FLASH; }
+
+ static class Image
+ {
+ public String uri;
+ public String title;
+ public int width;
+ public int height;
+ public Size size;
+
+ public Image() { }
+ public Image(String uri, String title, int w, int h, Size s)
+ {
+ this.uri = uri;
+ this.title = title;
+ width = w;
+ height = h;
+ size = s;
+ }
+ }
+
+ static class Content
+ {
+ public Player player;
+ public String uri;
+ public String title;
+ public int width;
+ public int height;
+ public String format;
+ public long duration;
+ public long size;
+ public int bitrate;
+ public String[] persons;
+ public String copyright;
+ }
+
+ /*
+ /**********************************************************
+ /* Unit tests
+ /**********************************************************
+ */
+
+ public void testSimple() throws IOException
+ {
+ final String STR1 = "a";
+
+ byte[] data = _smileDoc("{ "+quote(STR1)+":1, \"foobar\":2, \"longername\":3 }");
+ SmileFactory f = new SmileFactory();
+ SmileParser p = _smileParser(f, data);
+ final BytesToNameCanonicalizer symbols1 = p._symbols;
+ assertEquals(0, symbols1.size());
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ // field names are interned:
+ assertSame(STR1, p.getCurrentName());
+ assertEquals(1, symbols1.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertSame("foobar", p.getCurrentName());
+ assertEquals(2, symbols1.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertSame("longername", p.getCurrentName());
+ assertEquals(3, symbols1.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ assertEquals(3, symbols1.size());
+ p.close();
+
+ // but let's verify that symbol table gets reused properly
+ p = _smileParser(f, data);
+ BytesToNameCanonicalizer symbols2 = p._symbols;
+ // symbol tables are not reused, but contents are:
+ assertNotSame(symbols1, symbols2);
+ assertEquals(3, symbols2.size());
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ // field names are interned:
+ assertSame(STR1, p.getCurrentName());
+ assertEquals(3, symbols2.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertSame("foobar", p.getCurrentName());
+ assertEquals(3, symbols2.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertSame("longername", p.getCurrentName());
+ assertEquals(3, symbols2.size());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ assertEquals(3, symbols2.size());
+ p.close();
+
+ assertEquals(3, symbols2.size());
+ p.close();
+ }
+
+ public void testSharedNames() throws IOException
+ {
+ final int COUNT = 19000;
+
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.WRITE_HEADER, false);
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_NAMES, true);
+ ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
+ JsonGenerator gen = f.createGenerator(out);
+ gen.writeStartArray();
+ Random rnd = new Random(COUNT);
+ for (int i = 0; i < COUNT; ++i) {
+ gen.writeStartObject();
+ int nr = rnd.nextInt() % 1200;
+ gen.writeNumberField("f"+nr, nr);
+ gen.writeEndObject();
+ }
+ gen.writeEndArray();
+ gen.close();
+ byte[] json = out.toByteArray();
+
+ // And verify
+ f.configure(SmileParser.Feature.REQUIRE_HEADER, false);
+ JsonParser jp = f.createParser(json);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ rnd = new Random(COUNT);
+ for (int i = 0; i < COUNT; ++i) {
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ int nr = rnd.nextInt() % 1200;
+ String name = "f"+nr;
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals(name, jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(nr, jp.getIntValue());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+
+ public void testSharedStrings() throws IOException
+ {
+ final int count = 19000;
+ byte[] baseline = writeStringValues(false, count);
+ assertEquals(763589, baseline.length);
+ verifyStringValues(baseline, count);
+
+ // and then shared; should be much smaller
+ byte[] shared = writeStringValues(true, count);
+ if (shared.length >= baseline.length) {
+ fail("Expected shared String length < "+baseline.length+", was "+shared.length);
+ }
+ verifyStringValues(shared, count);
+ }
+
+ public void testSharedStringsInArrays() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
+ JsonGenerator gen = f.createGenerator(out);
+ gen.writeStartArray();
+ for (String value : SHARED_SYMBOLS) {
+ gen.writeString(value);
+ }
+ gen.writeEndArray();
+ gen.close();
+
+ byte[] smile = out.toByteArray();
+
+ JsonParser jp = f.createParser(smile);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ for (String value : SHARED_SYMBOLS) {
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals(value, jp.getText());
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+
+ public void testSharedStringsInObject() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+ ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
+ JsonGenerator gen = f.createGenerator(out);
+ gen.writeStartObject();
+ for (int i = 0; i < SHARED_SYMBOLS.length; ++i) {
+ gen.writeFieldName("a"+i);
+ gen.writeString(SHARED_SYMBOLS[i]);
+ }
+ gen.writeEndObject();
+ gen.close();
+
+ byte[] smile = out.toByteArray();
+
+ JsonParser jp = f.createParser(smile);
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ for (int i = 0; i < SHARED_SYMBOLS.length; ++i) {
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("a"+i, jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals(SHARED_SYMBOLS[i], jp.getText());
+ }
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ }
+
+ public void testSharedStringsMixed() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
+ JsonGenerator gen = f.createGenerator(out);
+ gen.writeStartObject();
+
+ gen.writeFieldName("media");
+ gen.writeStartObject();
+
+ gen.writeStringField("uri", "g");
+ gen.writeStringField("title", "J");
+ gen.writeNumberField("width", 640);
+ gen.writeStringField("format", "v");
+ gen.writeFieldName("persons");
+ gen.writeStartArray();
+ gen.writeString("B");
+ gen.writeString("S");
+ gen.writeEndArray();
+ gen.writeStringField("player", "JAVA");
+ gen.writeStringField("copyright", "NONE");
+
+ gen.writeEndObject(); // media
+
+ gen.writeFieldName("images");
+ gen.writeStartArray();
+
+ // 3 instances of identical entries
+ for (int i = 0; i < 3; ++i) {
+ gen.writeStartObject();
+ gen.writeStringField("uri", "h");
+ gen.writeStringField("title", "J");
+ gen.writeNumberField("width", 1024);
+ gen.writeNumberField("height", 768);
+ gen.writeEndObject();
+ }
+ gen.writeEndArray();
+
+ gen.writeEndObject();
+ gen.close();
+
+ byte[] smile = out.toByteArray();
+
+ JsonParser jp = f.createParser(smile);
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("media", jp.getCurrentName());
+
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("uri", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("g", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("title", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("J", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("width", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(640, jp.getIntValue());
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("format", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("v", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("persons", jp.getCurrentName());
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("B", jp.getText());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("S", jp.getText());
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("player", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("JAVA", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("copyright", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("NONE", jp.getText());
+
+ assertToken(JsonToken.END_OBJECT, jp.nextToken()); // media
+
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("images", jp.getCurrentName());
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+
+ // 3 instances of identical entries:
+ for (int i = 0; i < 3; ++i) {
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("uri", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("h", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("title", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals("J", jp.getText());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("width", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(1024, jp.getIntValue());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("height", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(768, jp.getIntValue());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ }
+
+ assertToken(JsonToken.END_ARRAY, jp.nextToken()); // images
+
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ }
+
+ public void testDataBindingAndShared() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, true);
+ MediaItem item = new MediaItem();
+ Content c = new Content();
+ c.uri = "g";
+ c.title = "J";
+ c.width = 640;
+ c.height = 480;
+ c.format = "v";
+ c.duration = 18000000L;
+ c.size = 58982400L;
+ c.bitrate = 262144;
+ c.persons = new String[] { "B", "S" };
+ c.player = Player.JAVA;
+ c.copyright = "NONE";
+ item.content = c;
+ item.images = new Image[] {
+ new Image("h", "J", 1024, 768, Size.LARGE),
+ new Image("h", "J", 320, 240, Size.LARGE)
+ };
+
+ // Ok: let's just do quick comparison (yes/no)...
+ ObjectMapper plain = new ObjectMapper();
+ ObjectMapper smiley = new ObjectMapper(f);
+ String exp = plain.writeValueAsString(item);
+ byte[] smile = smiley.writeValueAsBytes(item);
+ MediaItem result = smiley.readValue(smile, 0, smile.length, MediaItem.class);
+ String actual = plain.writeValueAsString(result);
+ assertEquals(exp, actual);
+ }
+
+ /**
+ * Reproducing [JACKSON-561] (and [JACKSON-562])
+ */
+ public void testIssue562() throws IOException
+ {
+ JsonFactory factory = new SmileFactory();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ JsonGenerator gen = factory.createGenerator(bos);
+ gen.writeStartObject();
+ gen.writeFieldName("z_aaaabbbbccccddddee");
+ gen.writeString("end");
+ gen.writeFieldName("a_aaaabbbbccccddddee");
+ gen.writeString("start");
+ gen.writeEndObject();
+ gen.close();
+
+ JsonParser parser = factory.createParser(bos.toByteArray());
+ assertToken(JsonToken.START_OBJECT, parser.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals("z_aaaabbbbccccddddee", parser.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, parser.nextToken());
+ assertEquals("end", parser.getText());
+
+ // This one fails...
+ assertToken(JsonToken.FIELD_NAME, parser.nextToken());
+ assertEquals("a_aaaabbbbccccddddee", parser.getCurrentName());
+ assertToken(JsonToken.VALUE_STRING, parser.nextToken());
+ assertEquals("start", parser.getText());
+
+ assertToken(JsonToken.END_OBJECT, parser.nextToken());
+ }
+
+ /**
+ * Verification that [JACKSON-564] was fixed.
+ */
+ public void testIssue564() throws Exception
+ {
+ JsonFactory factory = new SmileFactory();
+
+ ByteArrayOutputStream bos1 = new ByteArrayOutputStream();
+ JsonGenerator generator = factory.createGenerator(bos1);
+ generator.writeStartObject();
+ generator.writeFieldName("query");
+ generator.writeStartObject();
+ generator.writeFieldName("term");
+ generator.writeStartObject();
+ generator.writeStringField("doc.payload.test_record_main.string_not_analyzed__s", "foo");
+ generator.writeEndObject();
+ generator.writeEndObject();
+ generator.writeEndObject();
+ generator.close();
+
+ JsonParser parser = factory.createParser(bos1.toByteArray());
+ JsonToken token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ assertEquals("query", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ assertEquals("term", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ assertEquals("doc.payload.test_record_main.string_not_analyzed__s", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.VALUE_STRING, token);
+ assertEquals("foo", parser.getText());
+ parser.close();
+
+ ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
+ generator = factory.createGenerator(bos2);
+ generator.writeStartObject();
+ generator.writeFieldName("query");
+ generator.writeStartObject();
+ generator.writeFieldName("term");
+ generator.writeStartObject();
+ // note the difference here, teh field is analyzed2 and not analyzed as in the first doc, as well
+ // as having a different value, though don't think it matters
+ generator.writeStringField("doc.payload.test_record_main.string_not_analyzed2__s", "bar");
+ generator.writeEndObject();
+ generator.writeEndObject();
+ generator.writeEndObject();
+ generator.close();
+
+ parser = factory.createParser(bos2.toByteArray());
+ token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ assertEquals("query", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ assertEquals("term", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.START_OBJECT, token);
+ token = parser.nextToken();
+ assertToken(JsonToken.FIELD_NAME, token);
+ // here we fail..., seems to be a problem with field caching factory level???
+ // since we get the field name of the previous (bos1) document field value (withou the 2)
+ assertEquals("doc.payload.test_record_main.string_not_analyzed2__s", parser.getCurrentName());
+ token = parser.nextToken();
+ assertToken(JsonToken.VALUE_STRING, token);
+ assertEquals("bar", parser.getText());
+
+ parser.close();
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private final String CHARS_40 = "0123456789012345678901234567890123456789";
+
+ private byte[] writeStringValues(boolean enableSharing, int COUNT) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.WRITE_HEADER, true);
+ f.configure(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES, enableSharing);
+ ByteArrayOutputStream out = new ByteArrayOutputStream(4000);
+ JsonGenerator gen = f.createGenerator(out);
+ gen.writeStartArray();
+ Random rnd = new Random(COUNT);
+ for (int i = 0; i < COUNT; ++i) {
+ gen.writeString(generateString(rnd.nextInt()));
+ }
+ gen.writeEndArray();
+ gen.close();
+ return out.toByteArray();
+ }
+
+ private void verifyStringValues(byte[] json, int COUNT) throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ JsonParser jp = f.createParser(json);
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ Random rnd = new Random(COUNT);
+ for (int i = 0; i < COUNT; ++i) {
+ String str = generateString(rnd.nextInt());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertEquals(str, jp.getText());
+ }
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+
+ private String generateString(int rawNr)
+ {
+ int nr = rawNr % 1100;
+ // Actually, let's try longer ones too
+ String str = "some kind of String value we use"+nr;
+ if (nr > 900) {
+ str += CHARS_40;
+ }
+ return str;
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserUnicode.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserUnicode.java
new file mode 100644
index 0000000..b72a0b9
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestParserUnicode.java
@@ -0,0 +1,27 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+public class TestParserUnicode extends SmileTestBase
+{
+ // [Issue-2]: probs with Surrogate handling
+ public void testLongUnicodeWithSurrogates() throws IOException
+ {
+ final String SURROGATE_CHARS = "\ud834\udd1e";
+ StringBuilder sb = new StringBuilder(300);
+ while (sb.length() < 300) {
+ sb.append(SURROGATE_CHARS);
+ }
+ final String TEXT = sb.toString();
+ byte[] data = _smileDoc(quote(TEXT));
+
+ SmileParser p = _smileParser(data);
+ assertNull(p.getCurrentToken());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(TEXT, p.getText());
+ assertNull(p.nextToken());
+ }
+
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileDetection.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileDetection.java
new file mode 100644
index 0000000..af670ea
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileDetection.java
@@ -0,0 +1,151 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.format.DataFormatDetector;
+import com.fasterxml.jackson.core.format.DataFormatMatcher;
+import com.fasterxml.jackson.core.format.MatchStrength;
+
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.fasterxml.jackson.dataformat.smile.SmileParser;
+
+public class TestSmileDetection extends SmileTestBase
+{
+
+ public void testSimpleObjectWithHeader() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ DataFormatDetector detector = new DataFormatDetector(f);
+ byte[] doc = _smileDoc("{\"a\":3}", true);
+ DataFormatMatcher matcher = detector.findFormat(doc);
+ // should have match
+ assertTrue(matcher.hasMatch());
+ assertEquals("Smile", matcher.getMatchedFormatName());
+ assertSame(f, matcher.getMatch());
+ // with header, should be full match
+ assertEquals(MatchStrength.FULL_MATCH, matcher.getMatchStrength());
+ // and so:
+ JsonParser jp = matcher.createParserWithMatch();
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("a", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(3, jp.getIntValue());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+
+ public void testSimpleObjectWithoutHeader() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ DataFormatDetector detector = new DataFormatDetector(f);
+ f.disable(SmileParser.Feature.REQUIRE_HEADER);
+ byte[] doc = _smileDoc("{\"abc\":false}", false);
+ DataFormatMatcher matcher = detector.findFormat(doc);
+ assertTrue(matcher.hasMatch());
+ assertEquals("Smile", matcher.getMatchedFormatName());
+ assertSame(f, matcher.getMatch());
+ assertEquals(MatchStrength.SOLID_MATCH, matcher.getMatchStrength());
+ JsonParser jp = matcher.createParserWithMatch();
+ assertToken(JsonToken.START_OBJECT, jp.nextToken());
+ assertToken(JsonToken.FIELD_NAME, jp.nextToken());
+ assertEquals("abc", jp.getCurrentName());
+ assertToken(JsonToken.VALUE_FALSE, jp.nextToken());
+ assertToken(JsonToken.END_OBJECT, jp.nextToken());
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+
+ public void testSimpleArrayWithHeader() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ DataFormatDetector detector = new DataFormatDetector(f);
+ byte[] doc = _smileDoc("[ true, 7 ]", true);
+ DataFormatMatcher matcher = detector.findFormat(doc);
+ // should have match
+ assertTrue(matcher.hasMatch());
+ assertEquals("Smile", matcher.getMatchedFormatName());
+ assertSame(f, matcher.getMatch());
+ // with header, should be full match
+ assertEquals(MatchStrength.FULL_MATCH, matcher.getMatchStrength());
+ JsonParser jp = matcher.createParserWithMatch();
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_TRUE, jp.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertEquals(7, jp.getIntValue());
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+
+ public void testSimpleArrayWithoutHeader() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.disable(SmileParser.Feature.REQUIRE_HEADER);
+ DataFormatDetector detector = new DataFormatDetector(f);
+ byte[] doc = _smileDoc("[ -13 ]", false);
+ DataFormatMatcher matcher = detector.findFormat(doc);
+ assertTrue(matcher.hasMatch());
+ assertEquals("Smile", matcher.getMatchedFormatName());
+ assertSame(f, matcher.getMatch());
+ assertEquals(MatchStrength.SOLID_MATCH, matcher.getMatchStrength());
+ JsonParser jp = matcher.createParserWithMatch();
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, jp.nextToken());
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+
+ /*
+ /**********************************************************
+ /* Simple negative tests
+ /**********************************************************
+ */
+
+ /*
+ * Also let's ensure no match is found if data doesn't support it...
+ * Let's use 0xFD since it can not be included (except in raw binary;
+ * use of which requires header to be present)
+ */
+ public void testSimpleInvalid() throws Exception
+ {
+ DataFormatDetector detector = new DataFormatDetector(new SmileFactory());
+ byte FD = (byte) 0xFD;
+ byte[] DOC = new byte[] { FD, FD, FD, FD };
+ DataFormatMatcher matcher = detector.findFormat(new ByteArrayInputStream(DOC));
+ assertFalse(matcher.hasMatch());
+ assertEquals(MatchStrength.INCONCLUSIVE, matcher.getMatchStrength());
+ assertNull(matcher.createParserWithMatch());
+ }
+
+ /*
+ /**********************************************************
+ /* Fallback tests to ensure Smile is found even against JSON
+ /**********************************************************
+ */
+
+ public void testSmileVsJson() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.disable(SmileParser.Feature.REQUIRE_HEADER);
+ DataFormatDetector detector = new DataFormatDetector(new JsonFactory(), f);
+ // to make it bit trickier, leave out header
+ byte[] doc = _smileDoc("[ \"abc\" ]", false);
+ DataFormatMatcher matcher = detector.findFormat(doc);
+ assertTrue(matcher.hasMatch());
+ assertEquals("Smile", matcher.getMatchedFormatName());
+ assertSame(f, matcher.getMatch());
+ // without header, just solid
+ assertEquals(MatchStrength.SOLID_MATCH, matcher.getMatchStrength());
+ JsonParser jp = matcher.createParserWithMatch();
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_STRING, jp.nextToken());
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ assertNull(jp.nextToken());
+ jp.close();
+ }
+
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileGeneratorBinary.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileGeneratorBinary.java
new file mode 100644
index 0000000..d5830e9
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileGeneratorBinary.java
@@ -0,0 +1,97 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import org.junit.Assert;
+
+import com.fasterxml.jackson.core.*;
+
+public class TestSmileGeneratorBinary extends SmileTestBase
+{
+ static class ThrottledInputStream extends FilterInputStream
+ {
+ protected final int _maxBytes;
+
+ public ThrottledInputStream(byte[] data, int maxBytes)
+ {
+ this(new ByteArrayInputStream(data), maxBytes);
+ }
+
+ public ThrottledInputStream(InputStream in, int maxBytes)
+ {
+ super(in);
+ _maxBytes = maxBytes;
+ }
+
+ @Override
+ public int read(byte[] buf) throws IOException {
+ return read(buf, 0, buf.length);
+ }
+
+ @Override
+ public int read(byte[] buf, int offset, int len) throws IOException {
+ return in.read(buf, offset, Math.min(_maxBytes, len));
+ }
+
+ }
+
+ /*
+ /**********************************************************
+ /* Test methods
+ /**********************************************************
+ */
+
+ public void testStreamingBinary() throws Exception
+ {
+ _testStreamingBinary(true);
+ _testStreamingBinary(false);
+ }
+
+ public void testBinaryWithoutLength() throws Exception
+ {
+ final SmileFactory f = new SmileFactory();
+ JsonGenerator jg = f.createGenerator(new ByteArrayOutputStream());
+ try {
+ jg.writeBinary(new ByteArrayInputStream(new byte[1]), -1);
+ fail("Should have failed");
+ } catch (UnsupportedOperationException e) {
+ verifyException(e, "must pass actual length");
+ }
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private final static String TEXT = "Some content so that we can test encoding of base64 data; must"
+ +" be long enough include a line wrap or two...";
+ private final static String TEXT4 = TEXT + TEXT + TEXT + TEXT;
+
+ private void _testStreamingBinary(boolean rawBinary) throws Exception
+ {
+ final SmileFactory f = new SmileFactory();
+ f.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, !rawBinary);
+
+ final byte[] INPUT = TEXT4.getBytes("UTF-8");
+ for (int chunkSize : new int[] { 1, 2, 3, 4, 7, 11, 29, 5000 }) {
+ JsonGenerator jgen;
+
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ jgen = f.createGenerator(bytes);
+ jgen.writeStartArray();
+ InputStream data = new ThrottledInputStream(INPUT, chunkSize);
+ jgen.writeBinary(data, INPUT.length);
+ jgen.writeEndArray();
+ jgen.close();
+
+ JsonParser jp = f.createParser(bytes.toByteArray());
+ assertToken(JsonToken.START_ARRAY, jp.nextToken());
+ assertToken(JsonToken.VALUE_EMBEDDED_OBJECT, jp.nextToken());
+ byte[] b = jp.getBinaryValue();
+ Assert.assertArrayEquals(INPUT, b);
+ assertToken(JsonToken.END_ARRAY, jp.nextToken());
+ }
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileUtil.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileUtil.java
new file mode 100644
index 0000000..1c3aed1
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestSmileUtil.java
@@ -0,0 +1,44 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import com.fasterxml.jackson.dataformat.smile.SmileUtil;
+
+public class TestSmileUtil extends SmileTestBase
+{
+ /**
+ * Verification of helper methods used to handle with zigzag encoding
+ */
+ public void testZigZagInt()
+ {
+ // simple encode
+ assertEquals(0, SmileUtil.zigzagEncode(0));
+ assertEquals(1, SmileUtil.zigzagEncode(-1));
+ assertEquals(2, SmileUtil.zigzagEncode(1));
+ assertEquals(0xFFFFFFFF, SmileUtil.zigzagEncode(Integer.MIN_VALUE));
+ assertEquals(0xFFFFFFFE, SmileUtil.zigzagEncode(Integer.MAX_VALUE));
+
+ // simple decode
+ assertEquals(0, SmileUtil.zigzagDecode(0));
+ assertEquals(-1, SmileUtil.zigzagDecode(1));
+ assertEquals(1, SmileUtil.zigzagDecode(2));
+ assertEquals(0x7fffFFFF, SmileUtil.zigzagDecode(0xFFFFFFFE));
+ assertEquals(Integer.MIN_VALUE, SmileUtil.zigzagDecode(0xFFFFFFFF));
+
+ // round-trip
+ assertEquals(Integer.MIN_VALUE, SmileUtil.zigzagDecode(SmileUtil.zigzagEncode(Integer.MIN_VALUE)));
+ assertEquals(Integer.MAX_VALUE, SmileUtil.zigzagDecode(SmileUtil.zigzagEncode(Integer.MAX_VALUE)));
+ }
+
+ public void testZigZagLong()
+ {
+ assertEquals(0L, SmileUtil.zigzagEncode(0L));
+ assertEquals(-1L, SmileUtil.zigzagEncode(Long.MIN_VALUE));
+ assertEquals(-2L, SmileUtil.zigzagEncode(Long.MAX_VALUE));
+
+ assertEquals(Long.MAX_VALUE, SmileUtil.zigzagDecode(-2L));
+ assertEquals(Long.MIN_VALUE, SmileUtil.zigzagDecode(-1L));
+
+ // round-trip
+ assertEquals(Long.MIN_VALUE, SmileUtil.zigzagDecode(SmileUtil.zigzagEncode(Long.MIN_VALUE)));
+ assertEquals(Long.MAX_VALUE, SmileUtil.zigzagDecode(SmileUtil.zigzagEncode(Long.MAX_VALUE)));
+}
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestTreeHandling.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestTreeHandling.java
new file mode 100644
index 0000000..354f0cb
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestTreeHandling.java
@@ -0,0 +1,37 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import org.junit.Assert;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class TestTreeHandling extends SmileTestBase
+{
+ public void testSimple() throws Exception
+ {
+ // create the serialized JSON with byte array
+ ObjectMapper mapper = new ObjectMapper(new SmileFactory());
+
+ ObjectNode top1 = mapper.createObjectNode();
+ ObjectNode foo1 = top1.putObject("foo");
+ foo1.put("bar", "baz");
+ final String TEXT = "Caf\u00e9 1\u20ac";
+ final byte[] TEXT_BYTES = TEXT.getBytes("UTF-8");
+ foo1.put("dat", TEXT_BYTES);
+
+ byte[] doc = mapper.writeValueAsBytes(top1);
+ // now, deserialize
+ JsonNode top2 = mapper.readValue(doc, JsonNode.class);
+ JsonNode foo2 = top2.get("foo");
+ assertEquals("baz", foo2.get("bar").textValue());
+
+ JsonNode datNode = foo2.get("dat");
+ if (!datNode.isBinary()) {
+ fail("Expected binary node; got "+datNode.getClass().getName());
+ }
+ byte[] bytes = datNode.binaryValue();
+ Assert.assertArrayEquals(TEXT_BYTES, bytes);
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/dataformat/smile/TestVersions.java b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestVersions.java
new file mode 100644
index 0000000..588a62c
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/dataformat/smile/TestVersions.java
@@ -0,0 +1,32 @@
+package com.fasterxml.jackson.dataformat.smile;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.Versioned;
+
+/**
+ * Tests to verify [JACKSON-278]
+ */
+public class TestVersions extends SmileTestBase
+{
+ public void testMapperVersions() throws IOException
+ {
+ SmileFactory f = new SmileFactory();
+ f.disable(SmileParser.Feature.REQUIRE_HEADER);
+ assertVersion(f);
+ assertVersion(f.createGenerator(new ByteArrayOutputStream()));
+ assertVersion(f.createParser(new byte[0]));
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void assertVersion(Versioned v)
+ {
+ assertEquals(PackageVersion.VERSION, v.version());
+ }
+}
+
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-java/jackson-dataformat-smile.git
More information about the pkg-java-commits
mailing list