mirror of
https://github.com/fankes/moshi.git
synced 2025-10-18 23:49:21 +08:00
Merge pull request #1222 from square/jwilson.0912.json_value_source
JsonValueSource: a brace-matching implementation of okio.Source
This commit is contained in:
Jesse Wilson
GitHub
208
moshi/src/main/java/com/squareup/moshi/JsonValueSource.java
Normal file
208
moshi/src/main/java/com/squareup/moshi/JsonValueSource.java
Normal file
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Square, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.squareup.moshi;
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
import okio.Buffer;
|
||||
import okio.BufferedSource;
|
||||
import okio.ByteString;
|
||||
import okio.Source;
|
||||
import okio.Timeout;
|
||||
|
||||
/**
|
||||
* This source reads a prefix of another source as a JSON value and then terminates. It can read
|
||||
* top-level arrays, objects, or strings only.
|
||||
*
|
||||
* <p>It implements {@linkplain JsonReader#setLenient lenient parsing} and has no mechanism to
|
||||
* enforce strict parsing. If the input is not valid or lenient JSON the behavior of this source is
|
||||
* unspecified.
|
||||
*/
|
||||
final class JsonValueSource implements Source {
|
||||
private static final ByteString STATE_JSON = ByteString.encodeUtf8("[]{}\"'/#");
|
||||
private static final ByteString STATE_SINGLE_QUOTED = ByteString.encodeUtf8("'\\");
|
||||
private static final ByteString STATE_DOUBLE_QUOTED = ByteString.encodeUtf8("\"\\");
|
||||
private static final ByteString STATE_END_OF_LINE_COMMENT = ByteString.encodeUtf8("\r\n");
|
||||
private static final ByteString STATE_C_STYLE_COMMENT = ByteString.encodeUtf8("*");
|
||||
private static final ByteString STATE_END_OF_JSON = ByteString.EMPTY;
|
||||
|
||||
private final BufferedSource source;
|
||||
private final Buffer buffer;
|
||||
|
||||
/**
|
||||
* The state indicates what kind of data is readable at {@link #limit}. This also serves
|
||||
* double-duty as the type of bytes we're interested in while in this state.
|
||||
*/
|
||||
private ByteString state = STATE_JSON;
|
||||
|
||||
/**
|
||||
* The level of nesting of arrays and objects. When the end of string, array, or object is
|
||||
* reached, this should be compared against 0. If it is zero, then we've read a complete value and
|
||||
* this source is exhausted.
|
||||
*/
|
||||
private int stackSize = 0;
|
||||
|
||||
/** The number of bytes immediately returnable to the caller. */
|
||||
private long limit = 0;
|
||||
|
||||
private boolean closed = false;
|
||||
|
||||
public JsonValueSource(BufferedSource source) {
|
||||
this.source = source;
|
||||
this.buffer = source.getBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Advance {@link #limit} until it is at least {@code byteCount} or the JSON object is complete.
|
||||
*
|
||||
* @throws EOFException if the stream is exhausted before the JSON object completes.
|
||||
*/
|
||||
private void advanceLimit(long byteCount) throws IOException {
|
||||
while (limit < byteCount) {
|
||||
// If we've finished the JSON object, we're done.
|
||||
if (state == STATE_END_OF_JSON) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If advancing requires more data in the buffer, grow it.
|
||||
if (limit == buffer.size()) {
|
||||
source.require(limit + 1L);
|
||||
}
|
||||
|
||||
// Find the next interesting character for the current state. If the buffer doesn't have one,
|
||||
// then we can read the entire buffer.
|
||||
long index = buffer.indexOfElement(state, limit);
|
||||
if (index == -1L) {
|
||||
limit = buffer.size();
|
||||
continue;
|
||||
}
|
||||
|
||||
byte b = buffer.getByte(index);
|
||||
|
||||
if (state == STATE_JSON) {
|
||||
switch (b) {
|
||||
case '[':
|
||||
case '{':
|
||||
stackSize++;
|
||||
limit = index + 1;
|
||||
break;
|
||||
|
||||
case ']':
|
||||
case '}':
|
||||
stackSize--;
|
||||
if (stackSize == 0) state = STATE_END_OF_JSON;
|
||||
limit = index + 1;
|
||||
break;
|
||||
|
||||
case '\"':
|
||||
state = STATE_DOUBLE_QUOTED;
|
||||
limit = index + 1;
|
||||
break;
|
||||
|
||||
case '\'':
|
||||
state = STATE_SINGLE_QUOTED;
|
||||
limit = index + 1;
|
||||
break;
|
||||
|
||||
case '/':
|
||||
source.require(index + 2);
|
||||
byte b2 = buffer.getByte(index + 1);
|
||||
if (b2 == '/') {
|
||||
state = STATE_END_OF_LINE_COMMENT;
|
||||
limit = index + 2;
|
||||
} else if (b2 == '*') {
|
||||
state = STATE_C_STYLE_COMMENT;
|
||||
limit = index + 2;
|
||||
} else {
|
||||
limit = index + 1;
|
||||
}
|
||||
break;
|
||||
|
||||
case '#':
|
||||
state = STATE_END_OF_LINE_COMMENT;
|
||||
limit = index + 1;
|
||||
break;
|
||||
}
|
||||
|
||||
} else if (state == STATE_SINGLE_QUOTED || state == STATE_DOUBLE_QUOTED) {
|
||||
if (b == '\\') {
|
||||
source.require(index + 2);
|
||||
limit = index + 2;
|
||||
} else {
|
||||
state = (stackSize > 0) ? STATE_JSON : STATE_END_OF_JSON;
|
||||
limit = index + 1;
|
||||
}
|
||||
|
||||
} else if (state == STATE_C_STYLE_COMMENT) {
|
||||
source.require(index + 2);
|
||||
if (buffer.getByte(index + 1) == '/') {
|
||||
limit = index + 2;
|
||||
state = STATE_JSON;
|
||||
} else {
|
||||
limit = index + 1;
|
||||
}
|
||||
|
||||
} else if (state == STATE_END_OF_LINE_COMMENT) {
|
||||
limit = index + 1;
|
||||
state = STATE_JSON;
|
||||
|
||||
} else {
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discards any remaining JSON data in this source that was left behind after it was closed. It is
|
||||
* an error to call {@link #read} after calling this method.
|
||||
*/
|
||||
public void discard() throws IOException {
|
||||
closed = true;
|
||||
while (state != STATE_END_OF_JSON) {
|
||||
advanceLimit(8192);
|
||||
source.skip(limit);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long read(Buffer sink, long byteCount) throws IOException {
|
||||
if (closed) throw new IllegalStateException("closed");
|
||||
if (byteCount == 0) return 0L;
|
||||
|
||||
advanceLimit(byteCount);
|
||||
|
||||
if (limit == 0) {
|
||||
if (state != STATE_END_OF_JSON) throw new AssertionError();
|
||||
return -1L;
|
||||
}
|
||||
|
||||
long result = Math.min(byteCount, limit);
|
||||
sink.write(buffer, result);
|
||||
limit -= result;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Timeout timeout() {
|
||||
return source.timeout();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
// Note that this does not close the underlying source; that's the creator's responsibility.
|
||||
closed = true;
|
||||
}
|
||||
}
|
||||
204
moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java
Normal file
204
moshi/src/test/java/com/squareup/moshi/JsonValueSourceTest.java
Normal file
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Square, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.squareup.moshi;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
import okio.Buffer;
|
||||
import org.junit.Test;
|
||||
|
||||
public final class JsonValueSourceTest {
|
||||
@Test
|
||||
public void simpleValues() throws IOException {
|
||||
assertThat(jsonPrefix("{\"hello\": \"world\"}, 1, 2, 3")).isEqualTo("{\"hello\": \"world\"}");
|
||||
assertThat(jsonPrefix("['hello', 'world'], 1, 2, 3")).isEqualTo("['hello', 'world']");
|
||||
assertThat(jsonPrefix("\"hello\", 1, 2, 3")).isEqualTo("\"hello\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void braceMatching() throws IOException {
|
||||
assertThat(jsonPrefix("[{},{},[],[{}]],[]")).isEqualTo("[{},{},[],[{}]]");
|
||||
assertThat(jsonPrefix("[\"a\",{\"b\":{\"c\":{\"d\":[\"e\"]}}}],[]"))
|
||||
.isEqualTo("[\"a\",{\"b\":{\"c\":{\"d\":[\"e\"]}}}]");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void stringEscapes() throws IOException {
|
||||
assertThat(jsonPrefix("[\"12\\u00334\"],[]")).isEqualTo("[\"12\\u00334\"]");
|
||||
assertThat(jsonPrefix("[\"12\\n34\"],[]")).isEqualTo("[\"12\\n34\"]");
|
||||
assertThat(jsonPrefix("[\"12\\\"34\"],[]")).isEqualTo("[\"12\\\"34\"]");
|
||||
assertThat(jsonPrefix("[\"12\\'34\"],[]")).isEqualTo("[\"12\\'34\"]");
|
||||
assertThat(jsonPrefix("[\"12\\\\34\"],[]")).isEqualTo("[\"12\\\\34\"]");
|
||||
assertThat(jsonPrefix("[\"12\\\\\"],[]")).isEqualTo("[\"12\\\\\"]");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bracesInStrings() throws IOException {
|
||||
assertThat(jsonPrefix("[\"]\"],[]")).isEqualTo("[\"]\"]");
|
||||
assertThat(jsonPrefix("[\"\\]\"],[]")).isEqualTo("[\"\\]\"]");
|
||||
assertThat(jsonPrefix("[\"\\[\"],[]")).isEqualTo("[\"\\[\"]");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void unterminatedString() throws IOException {
|
||||
try {
|
||||
jsonPrefix("{\"a\":\"b...");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void unterminatedObject() throws IOException {
|
||||
try {
|
||||
jsonPrefix("{\"a\":\"b\",\"c\":");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix("{");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void unterminatedArray() throws IOException {
|
||||
try {
|
||||
jsonPrefix("[\"a\",\"b\",\"c\",");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix("[");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientUnterminatedSingleQuotedString() throws IOException {
|
||||
try {
|
||||
jsonPrefix("{\"a\":'b...");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void emptyStream() throws IOException {
|
||||
try {
|
||||
jsonPrefix("");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix(" ");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix("/* comment */");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientSingleQuotedStrings() throws IOException {
|
||||
assertThat(jsonPrefix("['hello', 'world'], 1, 2, 3")).isEqualTo("['hello', 'world']");
|
||||
assertThat(jsonPrefix("'abc\\'', 123")).isEqualTo("'abc\\''");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientCStyleComments() throws IOException {
|
||||
assertThat(jsonPrefix("[\"a\"/* \"b\" */,\"c\"],[]")).isEqualTo("[\"a\"/* \"b\" */,\"c\"]");
|
||||
assertThat(jsonPrefix("[\"a\"/*]*/],[]")).isEqualTo("[\"a\"/*]*/]");
|
||||
assertThat(jsonPrefix("[\"a\"/**/],[]")).isEqualTo("[\"a\"/**/]");
|
||||
assertThat(jsonPrefix("[\"a\"/*/ /*/],[]")).isEqualTo("[\"a\"/*/ /*/]");
|
||||
assertThat(jsonPrefix("[\"a\"/*/ **/],[]")).isEqualTo("[\"a\"/*/ **/]");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientEndOfLineComments() throws IOException {
|
||||
assertThat(jsonPrefix("[\"a\"// \"b\" \n,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \n,\"c\"]");
|
||||
assertThat(jsonPrefix("[\"a\"// \"b\" \r\n,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \r\n,\"c\"]");
|
||||
assertThat(jsonPrefix("[\"a\"// \"b\" \r,\"c\"],[]")).isEqualTo("[\"a\"// \"b\" \r,\"c\"]");
|
||||
assertThat(jsonPrefix("[\"a\"//]\r\n\"c\"],[]")).isEqualTo("[\"a\"//]\r\n\"c\"]");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientSlashInToken() throws IOException {
|
||||
assertThat(jsonPrefix("{a/b:\"c\"},[]")).isEqualTo("{a/b:\"c\"}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientUnterminatedEndOfLineComment() throws IOException {
|
||||
try {
|
||||
jsonPrefix("{\"a\",//}");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lenientUnterminatedCStyleComment() throws IOException {
|
||||
try {
|
||||
jsonPrefix("{\"a\",/* *");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix("{\"a\",/* **");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
try {
|
||||
jsonPrefix("{\"a\",/* /**");
|
||||
fail();
|
||||
} catch (EOFException expected) {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void discard() throws IOException {
|
||||
Buffer allData = new Buffer();
|
||||
allData.writeUtf8("{\"a\",\"b\",\"c\"},[\"d\", \"e\"]");
|
||||
|
||||
JsonValueSource jsonValueSource = new JsonValueSource(allData);
|
||||
jsonValueSource.close();
|
||||
jsonValueSource.discard();
|
||||
|
||||
assertThat(allData.readUtf8()).isEqualTo(",[\"d\", \"e\"]");
|
||||
}
|
||||
|
||||
private String jsonPrefix(String string) throws IOException {
|
||||
Buffer allData = new Buffer();
|
||||
allData.writeUtf8(string);
|
||||
|
||||
Buffer jsonPrefixBuffer = new Buffer();
|
||||
jsonPrefixBuffer.writeAll(new JsonValueSource(allData));
|
||||
|
||||
String result = jsonPrefixBuffer.readUtf8();
|
||||
String remainder = allData.readUtf8();
|
||||
assertThat(result + remainder).isEqualTo(string);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user