Merge pull request #1224 from square/jwilson.0913.json_value_source

New public API, JsonReader.nextSource()

moshi/src/main/java/com/squareup/moshi/JsonReader.java

@@ -422,6 +422,49 @@ public abstract class JsonReader implements Closeable {
    */
   public abstract int nextInt() throws IOException;
 
+  /**
+   * Returns the next value as a stream of UTF-8 bytes and consumes it.
+   *
+   * <p>The following program demonstrates how JSON bytes are returned from an enclosing stream as
+   * their original bytes, including their original whitespace:
+   *
+   * <pre>{@code
+   * String json = "{\"a\": [4,  5  ,6.0, {\"x\":7}, 8], \"b\": 9}";
+   * JsonReader reader = JsonReader.of(new Buffer().writeUtf8(json));
+   * reader.beginObject();
+   * assertThat(reader.nextName()).isEqualTo("a");
+   * try (BufferedSource bufferedSource = reader.valueSource()) {
+   *   assertThat(bufferedSource.readUtf8()).isEqualTo("[4,  5  ,6.0, {\"x\":7}, 8]");
+   * }
+   * assertThat(reader.nextName()).isEqualTo("b");
+   * assertThat(reader.nextInt()).isEqualTo(9);
+   * reader.endObject();
+   * }</pre>
+   *
+   * <p>This reads an entire value: composite objects like arrays and objects are returned in their
+   * entirety. The stream starts with the first character of the value (typically {@code [}, <code>{
+   * </code>, or {@code "}) and ends with the last character of the object (typically {@code ]},
+   * <code>}</code>, or {@code "}).
+   *
+   * <p>The returned source may not be used after any other method on this {@code JsonReader} is
+   * called. For example, the following code crashes with an exception:
+   *
+   * <pre>{@code
+   * JsonReader reader = ...
+   * reader.beginArray();
+   * BufferedSource source = reader.valueSource();
+   * reader.endArray();
+   * source.readUtf8(); // Crash!
+   * }</pre>
+   *
+   * <p>The returned bytes are not validated. This method assumes the stream is well-formed JSON and
+   * only attempts to find the value's boundary in the byte stream. It is the caller's
+   * responsibility to check that the returned byte stream is a valid JSON value.
+   *
+   * <p>Closing the returned source <strong>does not</strong> close this reader.
+   */
+  public abstract BufferedSource nextSource() throws IOException;
+
   /**
    * Skips the next value recursively. If it is an object or array, all nested elements are skipped.
    * This method is intended for use when the JSON token stream contains unrecognized or unhandled

moshi/src/main/java/com/squareup/moshi/JsonUtf8Reader.java

@@ -22,6 +22,7 @@ import javax.annotation.Nullable;
 import okio.Buffer;
 import okio.BufferedSource;
 import okio.ByteString;
+import okio.Okio;
 
 final class JsonUtf8Reader extends JsonReader {
   private static final long MIN_INCOMPLETE_INTEGER = Long.MIN_VALUE / 10;
@@ -89,6 +90,13 @@ final class JsonUtf8Reader extends JsonReader {
    */
   private @Nullable String peekedString;
 
+  /**
+   * If non-null, the most recent value read was {@link #readJsonValue()}. The caller may be
+   * mid-stream so it is necessary to call {@link JsonValueSource#discard} to get to the end of the
+   * current JSON value before proceeding.
+   */
+  private @Nullable JsonValueSource valueSource;
+
   JsonUtf8Reader(BufferedSource source) {
     if (source == null) {
       throw new NullPointerException("source == null");
@@ -233,6 +241,10 @@ final class JsonUtf8Reader extends JsonReader {
   }
 
   private int doPeek() throws IOException {
+    if (valueSource != null) {
+      valueSource.discard();
+      valueSource = null;
+    }
     int peekStack = scopes[stackSize - 1];
     if (peekStack == JsonScope.EMPTY_ARRAY) {
       scopes[stackSize - 1] = JsonScope.NONEMPTY_ARRAY;
@@ -1013,6 +1025,55 @@ final class JsonUtf8Reader extends JsonReader {
     pathNames[stackSize - 1] = "null";
   }
 
+  @Override
+  public BufferedSource nextSource() throws IOException {
+    int p = peeked;
+    if (p == PEEKED_NONE) {
+      p = doPeek();
+    }
+
+    int valueSourceStackSize = 0;
+    Buffer prefix = new Buffer();
+    ByteString state = JsonValueSource.STATE_END_OF_JSON;
+    if (p == PEEKED_BEGIN_ARRAY) {
+      prefix.writeUtf8("[");
+      state = JsonValueSource.STATE_JSON;
+      valueSourceStackSize++;
+    } else if (p == PEEKED_BEGIN_OBJECT) {
+      prefix.writeUtf8("{");
+      state = JsonValueSource.STATE_JSON;
+      valueSourceStackSize++;
+    } else if (p == PEEKED_DOUBLE_QUOTED) {
+      prefix.writeUtf8("\"");
+      state = JsonValueSource.STATE_DOUBLE_QUOTED;
+    } else if (p == PEEKED_SINGLE_QUOTED) {
+      prefix.writeUtf8("'");
+      state = JsonValueSource.STATE_SINGLE_QUOTED;
+    } else if (p == PEEKED_NUMBER || p == PEEKED_LONG || p == PEEKED_UNQUOTED) {
+      prefix.writeUtf8(nextString());
+    } else if (p == PEEKED_TRUE) {
+      prefix.writeUtf8("true");
+    } else if (p == PEEKED_FALSE) {
+      prefix.writeUtf8("false");
+    } else if (p == PEEKED_NULL) {
+      prefix.writeUtf8("null");
+    } else if (p == PEEKED_BUFFERED) {
+      String string = nextString();
+      try (JsonWriter jsonWriter = JsonWriter.of(prefix)) {
+        jsonWriter.value(string);
+      }
+    } else {
+      throw new JsonDataException("Expected a value but was " + peek() + " at path " + getPath());
+    }
+
+    valueSource = new JsonValueSource(source, prefix, state, valueSourceStackSize);
+    peeked = PEEKED_NONE;
+    pathIndices[stackSize - 1]++;
+    pathNames[stackSize - 1] = "null";
+
+    return Okio.buffer(valueSource);
+  }
+
   /**
    * Returns the next character in the stream that is neither whitespace nor a part of a comment.
    * When this returns, the returned character is always at {@code buffer.getByte(0)}.

moshi/src/main/java/com/squareup/moshi/JsonValueReader.java

@@ -24,6 +24,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
+import okio.Buffer;
+import okio.BufferedSource;
 
 /**
  * This class reads a JSON document by traversing a Java object comprising maps, lists, and JSON
@@ -343,6 +345,16 @@ final class JsonValueReader extends JsonReader {
     }
   }
 
+  @Override
+  public BufferedSource nextSource() throws IOException {
+    Object value = readJsonValue();
+    Buffer result = new Buffer();
+    try (JsonWriter jsonWriter = JsonWriter.of(result)) {
+      jsonWriter.jsonValue(value);
+    }
+    return result;
+  }
+
   @Override
   public JsonReader peekJson() {
     return new JsonValueReader(this);

moshi/src/main/java/com/squareup/moshi/JsonValueSource.java

@@ -32,37 +32,47 @@ import okio.Timeout;
  * unspecified.
  */
 final class JsonValueSource implements Source {
-  private static final ByteString STATE_JSON = ByteString.encodeUtf8("[]{}\"'/#");
-  private static final ByteString STATE_SINGLE_QUOTED = ByteString.encodeUtf8("'\\");
-  private static final ByteString STATE_DOUBLE_QUOTED = ByteString.encodeUtf8("\"\\");
-  private static final ByteString STATE_END_OF_LINE_COMMENT = ByteString.encodeUtf8("\r\n");
-  private static final ByteString STATE_C_STYLE_COMMENT = ByteString.encodeUtf8("*");
-  private static final ByteString STATE_END_OF_JSON = ByteString.EMPTY;
+  static final ByteString STATE_JSON = ByteString.encodeUtf8("[]{}\"'/#");
+  static final ByteString STATE_SINGLE_QUOTED = ByteString.encodeUtf8("'\\");
+  static final ByteString STATE_DOUBLE_QUOTED = ByteString.encodeUtf8("\"\\");
+  static final ByteString STATE_END_OF_LINE_COMMENT = ByteString.encodeUtf8("\r\n");
+  static final ByteString STATE_C_STYLE_COMMENT = ByteString.encodeUtf8("*");
+  static final ByteString STATE_END_OF_JSON = ByteString.EMPTY;
 
   private final BufferedSource source;
   private final Buffer buffer;
 
+  /** If non-empty, data from this should be returned before data from {@link #source}. */
+  private final Buffer prefix;
+
   /**
    * The state indicates what kind of data is readable at {@link #limit}. This also serves
    * double-duty as the type of bytes we're interested in while in this state.
    */
-  private ByteString state = STATE_JSON;
+  private ByteString state;
 
   /**
    * The level of nesting of arrays and objects. When the end of string, array, or object is
    * reached, this should be compared against 0. If it is zero, then we've read a complete value and
    * this source is exhausted.
    */
-  private int stackSize = 0;
+  private int stackSize;
 
   /** The number of bytes immediately returnable to the caller. */
   private long limit = 0;
 
   private boolean closed = false;
 
-  public JsonValueSource(BufferedSource source) {
+  JsonValueSource(BufferedSource source) {
+    this(source, new Buffer(), STATE_JSON, 0);
+  }
+
+  JsonValueSource(BufferedSource source, Buffer prefix, ByteString state, int stackSize) {
     this.source = source;
     this.buffer = source.getBuffer();
+    this.prefix = prefix;
+    this.state = state;
+    this.stackSize = stackSize;
   }
 
   /**
@@ -182,6 +192,14 @@ final class JsonValueSource implements Source {
     if (closed) throw new IllegalStateException("closed");
     if (byteCount == 0) return 0L;
 
+    // If this stream has a prefix, consume that first.
+    if (!prefix.exhausted()) {
+      long prefixResult = prefix.read(sink, byteCount);
+      byteCount -= prefixResult;
+      long suffixResult = read(sink, byteCount);
+      return suffixResult != -1L ? suffixResult + prefixResult : prefixResult;
+    }
+
     advanceLimit(byteCount);
 
     if (limit == 0) {

moshi/src/test/java/com/squareup/moshi/FlattenTest.java

@@ -18,6 +18,7 @@ package com.squareup.moshi;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import org.junit.Test;
@@ -280,7 +281,7 @@ public final class FlattenTest {
   }
 
   @Test
-  public void flattenTopLevel() {
+  public void flattenTopLevel() throws IOException {
     JsonWriter writer = factory.newWriter();
     try {
       writer.beginFlatten();

moshi/src/test/java/com/squareup/moshi/JsonCodecFactory.java

@@ -19,6 +19,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import okio.Buffer;
+import okio.BufferedSource;
 
 abstract class JsonCodecFactory {
   private static final Moshi MOSHI = new Moshi.Builder().build();
@@ -123,12 +124,12 @@ abstract class JsonCodecFactory {
           }
 
           @Override
-          JsonWriter newWriter() {
+          JsonWriter newWriter() throws IOException {
             return value.newWriter();
           }
 
           @Override
-          String json() {
+          String json() throws IOException {
             return value.json();
           }
 
@@ -144,18 +145,55 @@ abstract class JsonCodecFactory {
           }
         };
 
-    return Arrays.asList(new Object[] {utf8}, new Object[] {value}, new Object[] {valuePeek});
+    /** Wrap the enclosing JsonReader or JsonWriter in another stream. */
+    final JsonCodecFactory valueSource =
+        new JsonCodecFactory() {
+          private JsonWriter writer;
+
+          @Override
+          public JsonReader newReader(String json) throws IOException {
+            JsonReader wrapped = utf8.newReader(json);
+            wrapped.setLenient(true);
+            BufferedSource valueSource = wrapped.nextSource();
+            return JsonReader.of(valueSource);
+          }
+
+          @Override
+          JsonWriter newWriter() throws IOException {
+            JsonWriter wrapped = utf8.newWriter();
+            wrapped.setLenient(true);
+            writer = JsonWriter.of(wrapped.valueSink());
+            return writer;
+          }
+
+          @Override
+          String json() throws IOException {
+            writer.close();
+            return utf8.json();
+          }
+
+          @Override
+          public String toString() {
+            return "ValueSource";
+          }
+        };
+
+    return Arrays.asList(
+        new Object[] {utf8},
+        new Object[] {value},
+        new Object[] {valuePeek},
+        new Object[] {valueSource});
   }
 
   abstract JsonReader newReader(String json) throws IOException;
 
-  abstract JsonWriter newWriter();
+  abstract JsonWriter newWriter() throws IOException;
 
   boolean implementsStrictPrecision() {
     return true;
   }
 
-  abstract String json();
+  abstract String json() throws IOException;
 
   boolean encodesToBytes() {
     return false;

moshi/src/test/java/com/squareup/moshi/JsonReaderTest.java

@@ -31,6 +31,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import okio.BufferedSource;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -1338,6 +1339,100 @@ public final class JsonReaderTest {
     }
   }
 
+  @Test
+  public void nextSourceString() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":\"this is a string\"}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("\"this is a string\"");
+    }
+  }
+
+  @Test
+  public void nextSourceLong() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":-2.0}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("-2.0");
+    }
+  }
+
+  @Test
+  public void nextSourceNull() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":null}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("null");
+    }
+  }
+
+  @Test
+  public void nextSourceBoolean() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":false}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("false");
+    }
+  }
+
+  @Test
+  public void nextSourceObject() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":{\"b\":2.0,\"c\":3.0}}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("{\"b\":2.0,\"c\":3.0}");
+    }
+  }
+
+  @Test
+  public void nextSourceArray() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":[2.0,2.0,3.0]}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("[2.0,2.0,3.0]");
+    }
+  }
+
+  /**
+   * When we call {@link JsonReader#selectString} it causes the reader to consume bytes of the input
+   * string. When attempting to read it as a stream afterwards the bytes are reconstructed.
+   */
+  @Test
+  public void nextSourceStringBuffered() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":\"b\"}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    assertThat(reader.selectString(JsonReader.Options.of("x'"))).isEqualTo(-1);
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("\"b\"");
+    }
+  }
+
+  /** If we don't read the bytes of the source, they JsonReader doesn't lose its place. */
+  @Test
+  public void nextSourceNotConsumed() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\"a\":\"b\",\"c\":\"d\"}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    reader.nextSource(); // Not closed.
+    assertThat(reader.nextName()).isEqualTo("c");
+    assertThat(reader.nextString()).isEqualTo("d");
+  }
+
   /** Peek a value, then read it, recursively. */
   private void readValue(JsonReader reader, boolean peekJsonFirst) throws IOException {
     JsonReader.Token token = reader.peek();

moshi/src/test/java/com/squareup/moshi/JsonUtf8ReaderTest.java

@@ -35,6 +35,7 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.util.Arrays;
 import okio.Buffer;
+import okio.BufferedSource;
 import okio.ForwardingSource;
 import okio.Okio;
 import org.junit.Ignore;
@@ -1385,4 +1386,26 @@ public final class JsonUtf8ReaderTest {
       }
     }
   }
+
+  @Test
+  public void nextSourceObject_withWhitespace() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\n  \"a\": {\n    \"b\": 2,\n    \"c\": 3\n  }\n}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("{\n    \"b\": 2,\n    \"c\": 3\n  }");
+    }
+  }
+
+  @Test
+  public void nextSourceLong_WithWhitespace() throws IOException {
+    // language=JSON
+    JsonReader reader = newReader("{\n  \"a\": -2\n}");
+    reader.beginObject();
+    assertThat(reader.nextName()).isEqualTo("a");
+    try (BufferedSource valueSource = reader.nextSource()) {
+      assertThat(valueSource.readUtf8()).isEqualTo("-2");
+    }
+  }
 }